diff --git a/.gitignore b/.gitignore index c181a2e4..18d136b7 100644 --- a/.gitignore +++ b/.gitignore @@ -59,3 +59,4 @@ crash-* leak-* timeout-* oom-* +.superpowers/ diff --git a/Makefile b/Makefile index 08b07011..72c230b9 100644 --- a/Makefile +++ b/Makefile @@ -50,8 +50,16 @@ RAY_MARCH ?= native DEBUG_CFLAGS = -fPIC $(WARNS) -std=$(STD) -g -O0 -march=$(RAY_MARCH) -DDEBUG \ -fsanitize=address,undefined -fno-omit-frame-pointer RELEASE_CFLAGS = -fPIC $(WARNS) -std=$(STD) -O3 -march=$(RAY_MARCH) \ - -funroll-loops -fomit-frame-pointer -fno-math-errno \ + -funroll-loops -fomit-frame-pointer -fno-math-errno -falign-functions=64 \ -fassociative-math -ffp-contract=fast -fno-signed-zeros -fno-trapping-math +# -falign-functions=64: start every function on a cache line. The default +# 16-byte alignment makes hot leaf functions' cost depend on where unrelated +# edits elsewhere in the same translation unit happen to push them — +# measured as a reproducible 14% swing on ClickBench q33/q34 +# (exec_group_sp_dyn_emit, ~45% of those queries) from an edit that path +# never executes. That noise floor is larger than most real optimisations, +# so it hides regressions and manufactures phantom ones. Costs a few KB of +# padding. # -fassociative-math: license to reorder FP additions/multiplications. # Required for autovectorization of F64 reductions (sum/avg/dot). # Without it, scalar_sum_f64_fn at group.c:1666 is a serial latency @@ -111,7 +119,7 @@ TSAN_LDFLAGS = -fsanitize=thread HARDENED_CFLAGS = -fPIC $(WARNS) -std=$(STD) -O3 -march=$(RAY_MARCH) -g \ -fno-omit-frame-pointer -DRAY_HARDENED \ -funroll-loops -fno-math-errno -fassociative-math -ffp-contract=fast \ - -fno-signed-zeros -fno-trapping-math + -fno-signed-zeros -fno-trapping-math -falign-functions=64 CFLAGS = $(DEBUG_CFLAGS) LDFLAGS = $(DEBUG_LDFLAGS) diff --git a/docs/docs/storage/index.md b/docs/docs/storage/index.md index c5741660..62b72116 100644 --- a/docs/docs/storage/index.md +++ b/docs/docs/storage/index.md @@ -129,6 +129,9 @@ ray_t* trades = ray_read_parted("db", "trades"); !!! note "Rayfall builtin" Use `.db.parted.get` from Rayfall to load partitioned tables: `(.db.parted.get "db" 'trades)`. See the [Rayfall Storage Builtins](#rayfall-storage) section below. +!!! warning "`update` over a partitioned table materializes in memory" + `update` on a partitioned table flattens the whole table into memory first — the parted/`MAPCOMMON` columns cannot be mutated in place — so the result is an ordinary in-memory table. It is **not** written back to the store: re-reading the root returns the original values, and the returned table loses its parted / memory-mapped identity. + ### Partition Pruning The query optimizer recognizes predicates on the `MAPCOMMON` column and eliminates entire partitions from the scan plan. This means a query filtering on a single date in a year of data only touches 1/365th of the files on disk — with zero per-row cost for the pruned partitions. diff --git a/src/core/platform.c b/src/core/platform.c index 92c1717d..e1e835af 100644 --- a/src/core/platform.c +++ b/src/core/platform.c @@ -39,6 +39,10 @@ #include #include #include +#include +#if defined(RAY_OS_MACOS) +#include /* sysctlbyname — hw.physicalcpu */ +#endif #include "mem/sys.h" /* -------------------------------------------------------------------------- @@ -247,6 +251,49 @@ uint32_t ray_thread_count(void) { return (n > 0) ? (uint32_t)n : 1; } +/* Physical cores (SMT siblings collapsed). The worker pool's kernels are + * memory-bound; two hyperthreads sharing one core's load/store machinery + * only add contention (measured: the full ClickBench suite runs ~11% + * SLOWER with 32 SMT threads than with the 16 physical cores on a 5950X). + * Counts unique (package, core) pairs from sysfs; any read failure falls + * back to the logical count so exotic systems keep the old behavior. */ +uint32_t ray_physical_core_count(void) { +#if defined(RAY_OS_MACOS) + int phys = 0; + size_t len = sizeof(phys); + if (sysctlbyname("hw.physicalcpu", &phys, &len, NULL, 0) == 0 && phys > 0) + return (uint32_t)phys; + return ray_thread_count(); +#else + uint32_t logical = ray_thread_count(); + /* (package_id << 16) | core_id per cpu; count distinct values. */ + enum { MAX_IDS = 4096 }; + uint32_t seen[MAX_IDS]; + uint32_t n_seen = 0; + for (uint32_t cpu = 0; cpu < logical && cpu < MAX_IDS; cpu++) { + char path[128]; + long core = -1, pkg = 0; + FILE* f; + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%u/topology/core_id", cpu); + f = fopen(path, "r"); + if (!f) return logical; /* no topology → fall back */ + if (fscanf(f, "%ld", &core) != 1) { fclose(f); return logical; } + fclose(f); + snprintf(path, sizeof(path), + "/sys/devices/system/cpu/cpu%u/topology/physical_package_id", + cpu); + f = fopen(path, "r"); + if (f) { if (fscanf(f, "%ld", &pkg) != 1) pkg = 0; fclose(f); } + uint32_t id = ((uint32_t)pkg << 16) | ((uint32_t)core & 0xFFFF); + uint32_t j = 0; + while (j < n_seen && seen[j] != id) j++; + if (j == n_seen) seen[n_seen++] = id; + } + return n_seen > 0 ? n_seen : logical; +#endif +} + /* -------------------------------------------------------------------------- * Semaphore * -------------------------------------------------------------------------- */ @@ -433,6 +480,12 @@ uint32_t ray_thread_count(void) { return (uint32_t)si.dwNumberOfProcessors; } +/* Windows: no cheap topology read here — fall back to the logical count + * (the SMT-aware default sizing is a POSIX-side optimization). */ +uint32_t ray_physical_core_count(void) { + return ray_thread_count(); +} + /* -------------------------------------------------------------------------- * Semaphore * -------------------------------------------------------------------------- */ diff --git a/src/core/platform.h b/src/core/platform.h index 2f40cb50..6f2a9882 100644 --- a/src/core/platform.h +++ b/src/core/platform.h @@ -173,6 +173,9 @@ bool ray_vm_hugepage(void* ptr, size_t size); ray_err_t ray_thread_create(ray_thread_t* t, ray_thread_fn fn, void* arg); ray_err_t ray_thread_join(ray_thread_t t); uint32_t ray_thread_count(void); +/* Physical cores (SMT siblings collapsed); falls back to the logical + * count when topology is unavailable. */ +uint32_t ray_physical_core_count(void); void ray_parallel_begin(void); void ray_parallel_end(void); diff --git a/src/core/pool.c b/src/core/pool.c index 91f92aa4..f7977ab9 100644 --- a/src/core/pool.c +++ b/src/core/pool.c @@ -147,7 +147,13 @@ static ray_err_t ray_pool_create_impl(ray_pool_t* pool, uint32_t n_workers, long v = strtol(env, NULL, 10); n_workers = (v > 0) ? (uint32_t)v : 0; } else { - uint32_t ncpu = ray_thread_count(); + /* Physical cores, not SMT threads: the pool's kernels are + * memory-bound, and hyperthread pairs sharing one core's + * load/store machinery only contend (full ClickBench suite + * measured ~11% slower at 32 SMT threads than at the 16 + * physical cores of a 5950X). ray_physical_core_count falls + * back to the logical count when topology is unreadable. */ + uint32_t ncpu = ray_physical_core_count(); n_workers = (ncpu > 1) ? ncpu - 1 : 0; } } diff --git a/src/io/csv.c b/src/io/csv.c index ab7e1641..153c128f 100644 --- a/src/io/csv.c +++ b/src/io/csv.c @@ -1466,14 +1466,17 @@ static int csv_hash_elem_size(int8_t t) { * BOOL/U8/I16 where the index would dwarf the column. * * Returns 1 to attach, 0 to skip. */ -static int csv_should_attach_hash(ray_t* v) { - if (!v || RAY_IS_ERR(v)) return 0; - int esz = csv_hash_elem_size(v->type); +/* Payload-level core of the hash-upgrade decision, shared with the + * .csv.splayed index builder (ray_splay_build_indexes) so the on-disk + * store makes the SAME hash-vs-zone decision the in-memory load does -- + * a reloaded store must not query slower than a fresh .csv.read. */ +int ray_csv_hash_upgrade_check(int8_t type, int64_t len, + const void* index_payload) { + const ray_index_t* ix = (const ray_index_t*)index_payload; + int esz = csv_hash_elem_size(type); if (esz == 0) return 0; - /* Need a chunk_zone we can read for entropy estimation. */ - if (!(v->attrs & RAY_ATTR_HAS_INDEX) || !v->index) return 0; - ray_index_t* ix = ray_index_payload(v->index); - if (ix->kind != RAY_IDX_CHUNK_ZONE || ix->u.chunk_zone.is_f64) return 0; + if (!ix || ix->kind != RAY_IDX_CHUNK_ZONE || ix->u.chunk_zone.is_f64) + return 0; uint32_t n_chunks = ix->u.chunk_zone.n_chunks; if (n_chunks < 4) return 0; const int64_t* mins = (const int64_t*)ray_data(ix->u.chunk_zone.mins); @@ -1535,7 +1538,7 @@ static int csv_should_attach_hash(ray_t* v) { * index dwarfs the data) out of the index set while admitting * I32 / I64 numeric IDs. Done in int64 arithmetic (we cap n * to anything that would overflow at the row counts we accept). */ - int64_t n = v->len; + int64_t n = len; if (n <= 0) return 0; uint64_t cap = 8; uint64_t want = (uint64_t)(2 * n); @@ -1547,6 +1550,13 @@ static int csv_should_attach_hash(ray_t* v) { return 1; } +static int csv_should_attach_hash(ray_t* v) { + if (!v || RAY_IS_ERR(v)) return 0; + if (!(v->attrs & RAY_ATTR_HAS_INDEX) || !v->index) return 0; + return ray_csv_hash_upgrade_check(v->type, v->len, + ray_index_payload(v->index)); +} + /* -------------------------------------------------------------------------- * `INT` schema columns — auto narrowest integer width. * @@ -1953,9 +1963,19 @@ static ray_t* csv_materialize_rows(const char* buf, size_t file_size, * After the chunk_zone attaches we re-walk the same columns and * upgrade the high-entropy ones to a hash index (the chunk_zone * stays as well — it's the entropy signal we just measured). See - * csv_should_attach_hash for the selectivity + memory cap. */ + * csv_should_attach_hash for the selectivity + memory cap. + * + * Progress: same treatment as the finalize dispatch above — the + * per-column index builds run their own pool dispatches whose row + * totals are not n_rows, so letting them drive the progress pump + * resets a completed parse to arbitrary fractions (visibly + * 100% -> 50% -> ...). Suppress progress for the whole index + * phase; the parse's completed row count stays on screen. */ + uint32_t idx_qmode = ray_qstats_mode(); + ray_qstats_set_mode(idx_qmode & ~RAY_QS_PROGRESS); for (int c = 0; c < ncols; c++) { if (ray_interrupted()) { + ray_qstats_set_mode(idx_qmode); for (int j = 0; j < ncols; j++) ray_release(col_vecs[j]); return ray_error("cancel", "interrupted"); } @@ -1964,6 +1984,7 @@ static ray_t* csv_materialize_rows(const char* buf, size_t file_size, if (v->len < (1 << 16)) continue; /* < one chunk, skip */ ray_t* r = ray_index_attach_chunk_zone(&v, 16); if (ray_interrupted()) { + ray_qstats_set_mode(idx_qmode); for (int j = 0; j < ncols; j++) ray_release(col_vecs[j]); return ray_error("cancel", "interrupted"); } @@ -1972,6 +1993,7 @@ static ray_t* csv_materialize_rows(const char* buf, size_t file_size, } for (int c = 0; c < ncols; c++) { if (ray_interrupted()) { + ray_qstats_set_mode(idx_qmode); for (int j = 0; j < ncols; j++) ray_release(col_vecs[j]); return ray_error("cancel", "interrupted"); } @@ -1984,11 +2006,13 @@ static ray_t* csv_materialize_rows(const char* buf, size_t file_size, * anyway, so the chunk_zone is dead weight. */ ray_t* r = ray_index_attach_hash(&v); if (ray_interrupted()) { + ray_qstats_set_mode(idx_qmode); for (int j = 0; j < ncols; j++) ray_release(col_vecs[j]); return ray_error("cancel", "interrupted"); } if (r && !RAY_IS_ERR(r)) col_vecs[c] = v; } + ray_qstats_set_mode(idx_qmode); ray_t* tbl = ray_table_new(ncols); if (!tbl || RAY_IS_ERR(tbl)) { @@ -2459,24 +2483,30 @@ ray_t* ray_read_csv_named_opts(const char* path, char delimiter, bool header, * matching loop in build_table_from_cols) — unsupported types * fall through to the unindexed path inside the consumer. * Second pass upgrades high-entropy columns to a hash index; - * see csv_should_attach_hash. */ + * see csv_should_attach_hash. + * Progress suppressed for the whole phase (same rationale as the + * finalize dispatch): the index builds' pool dispatches would + * reset the completed parse progress to arbitrary fractions. */ + uint32_t idx_qmode = ray_qstats_mode(); + ray_qstats_set_mode(idx_qmode & ~RAY_QS_PROGRESS); for (int c = 0; c < ncols; c++) { - if (ray_interrupted()) goto fail_cols_cancel; + if (ray_interrupted()) { ray_qstats_set_mode(idx_qmode); goto fail_cols_cancel; } ray_t* v = col_vecs[c]; if (!v || RAY_IS_ERR(v)) continue; if (v->len < (1 << 16)) continue; ray_t* r = ray_index_attach_chunk_zone(&v, 16); - if (ray_interrupted()) goto fail_cols_cancel; + if (ray_interrupted()) { ray_qstats_set_mode(idx_qmode); goto fail_cols_cancel; } if (r && !RAY_IS_ERR(r)) col_vecs[c] = v; } for (int c = 0; c < ncols; c++) { - if (ray_interrupted()) goto fail_cols_cancel; + if (ray_interrupted()) { ray_qstats_set_mode(idx_qmode); goto fail_cols_cancel; } ray_t* v = col_vecs[c]; if (!csv_should_attach_hash(v)) continue; ray_t* r = ray_index_attach_hash(&v); - if (ray_interrupted()) goto fail_cols_cancel; + if (ray_interrupted()) { ray_qstats_set_mode(idx_qmode); goto fail_cols_cancel; } if (r && !RAY_IS_ERR(r)) col_vecs[c] = v; } + ray_qstats_set_mode(idx_qmode); ray_t* tbl = ray_table_new(ncols); if (!tbl || RAY_IS_ERR(tbl)) { diff --git a/src/io/csv.h b/src/io/csv.h index 6b6b4c27..9b31bdaa 100644 --- a/src/io/csv.h +++ b/src/io/csv.h @@ -85,4 +85,10 @@ ray_err_t ray_csv_save_splayed_named_opts(const char* path, char delimiter, bool const char* dir, int64_t rows_per_chunk); ray_err_t ray_write_csv(ray_t* table, const char* path); +/* Hash-vs-chunk-zone index policy for a column, from its chunk-zone entropy + * (payload-level; see csv.c). Shared by the in-memory CSV load and the + * .csv.splayed store index builder so both make the same decision. */ +int ray_csv_hash_upgrade_check(int8_t type, int64_t len, + const void* index_payload); + #endif /* RAY_CSV_H */ diff --git a/src/lang/cal.h b/src/lang/cal.h index f3adc0e1..2111fbb3 100644 --- a/src/lang/cal.h +++ b/src/lang/cal.h @@ -41,20 +41,22 @@ static inline int date_leap_year(int year) { return (year % 4 == 0 && year % 100 != 0) || year % 400 == 0; } -static inline int32_t date_years_by_days(int yy) { - return (int32_t)((int64_t)yy * 365 + yy / 4 - yy / 100 + yy / 400); +static inline int64_t date_years_by_days(int yy) { + return (int64_t)yy * 365 + yy / 4 - yy / 100 + yy / 400; } /* Decode: days-since-epoch → year/month/day */ static inline void date_to_ymd(int32_t days, int* y, int* m, int* d) { - int32_t offset = days + date_years_by_days(RAY_DATE_EPOCH - 1); + /* int64 throughout: days near INT32_MAX overflow int32 after the epoch + * offset (e.g. (as 'DATE 2147483646)), producing garbage years. */ + int64_t offset = (int64_t)days + date_years_by_days(RAY_DATE_EPOCH - 1); double approx = (double)offset / 365.2425; int32_t years = (int32_t)(approx >= 0.0 ? approx + 0.5 : approx - 0.5); if (date_years_by_days(years) > offset) years -= 1; - int32_t rem = offset - date_years_by_days(years); + int64_t rem = offset - date_years_by_days(years); int yy = years + 1; int leap = date_leap_year(yy); int mid = 0; @@ -68,18 +70,18 @@ static inline void date_to_ymd(int32_t days, int* y, int* m, int* d) { *y = yy; *m = 1 + mid % 12; - *d = 1 + rem - (int32_t)MONTHDAYS[leap][mid]; + *d = 1 + (int)(rem - (int32_t)MONTHDAYS[leap][mid]); } /* Encode: year/month/day → days-since-epoch */ static inline int32_t ymd_to_date(int year, int month, int day) { int yy = (year > 0) ? year - 1 : 0; - int32_t ydays = date_years_by_days(yy); + int64_t ydays = date_years_by_days(yy); int leap = date_leap_year(year); int mm = (month > 0) ? month - 1 : 0; if (mm > 12) mm = 12; /* defensive: never index past the 13-wide table */ - int32_t mdays = (int32_t)MONTHDAYS[leap][mm]; - return ydays - date_years_by_days(RAY_DATE_EPOCH - 1) + mdays + day - 1; + int64_t mdays = (int32_t)MONTHDAYS[leap][mm]; + return (int32_t)(ydays - date_years_by_days(RAY_DATE_EPOCH - 1) + mdays + day - 1); } #endif /* RAY_CAL_H */ diff --git a/src/mem/heap.c b/src/mem/heap.c index 351a4a80..8c9fe9a3 100644 --- a/src/mem/heap.c +++ b/src/mem/heap.c @@ -1202,6 +1202,123 @@ static void* heap_direct_map_file(ray_heap_t* h, size_t map_size, return mapped; } +/* -------------------------------------------------------------------------- + * Direct-block reuse cache + * + * Analytical queries allocate the same few large scratch/result blocks + * every run (radix payload maps, order maps, result columns) and freed + * them straight back to the kernel: ~330MB of anon mmap+munmap round-trip + * per 10M-row group query, with the kernel re-zeroing every page on the + * next fault (kernel_init_pages was ~8% of ClickBench q17). Keep a small + * global stash of recently freed ANON direct blocks and serve size-matched + * requests from it. + * + * Accounting: a cached block's pages stay resident, so it remains counted + * in both the committed-RAM tracker and the g_anon_committed watermark; + * only the live-block stats (g_direct_bytes/count) drop on stash and rise + * again on reuse. Under watermark pressure the cache is drained before + * new memory is committed. File-backed spill blocks are never cached. + * + * Fit: first block with map_size in [need, need + max(need/4, 2MB)] — the + * repeat-query case hits exactly; the bound keeps waste at <= 25%. + * Concurrency: any thread may alloc or free a direct block, so a global + * spinlock guards the table — direct ops are a handful per query, so + * contention is nil. Budget: 1/16 of the anon watermark (physical RAM + * by default, the -m budget when set), capped at 512MB — no knob; the + * cache is invisible to correctness and self-drains under pressure. */ +#define RAY_DIRECT_CACHE_SLOTS 16 +typedef struct { void* base; size_t map_size; } ray_direct_cache_slot_t; +static ray_direct_cache_slot_t g_direct_cache[RAY_DIRECT_CACHE_SLOTS]; +static size_t g_direct_cache_bytes = 0; +static _Atomic(int) g_direct_cache_spin = 0; + +static inline void direct_cache_lock(void) { + while (atomic_exchange_explicit(&g_direct_cache_spin, 1, + memory_order_acquire)) { /* spin */ } +} +static inline void direct_cache_unlock(void) { + atomic_store_explicit(&g_direct_cache_spin, 0, memory_order_release); +} + +static size_t direct_cache_budget(void) { + int64_t wm = heap_anon_watermark(); + size_t b = (wm > 0) ? (size_t)wm / 16 : 0; + /* 1/16 of the watermark is the sole bound (1GB on a 16GB box, ~8GB + * on 128GB). Earlier absolute caps (512MB, then 4GB) each turned + * out to exclude exactly the blocks whose kernel re-zeroing cost the + * most at the next data scale: a 100M-row group query cycles several + * GB-scale scratch blocks (order map ~800MB, per-partition gather + * arrays ~1.6GB, result columns ~500MB) per execution, and every + * cache miss is an mmap + munmap + full page-zeroing round trip + * (measured 14-17% of q17/q18 wall as kernel_init_pages). */ + return b; +} + +/* Take a cached block whose map_size fits [need, need + waste bound]. + * Returns the base pointer (its map_size in *out_size) or NULL. */ +static void* direct_cache_take(size_t need, size_t* out_size) { + void* base = NULL; + direct_cache_lock(); + size_t slack = need / 4; + if (slack < (2u << 20)) slack = (2u << 20); + for (int i = 0; i < RAY_DIRECT_CACHE_SLOTS; i++) { + size_t ms = g_direct_cache[i].map_size; + if (g_direct_cache[i].base && ms >= need && ms - need <= slack) { + base = g_direct_cache[i].base; + *out_size = ms; + g_direct_cache[i].base = NULL; + g_direct_cache[i].map_size = 0; + g_direct_cache_bytes -= ms; + break; + } + } + direct_cache_unlock(); + return base; +} + +/* Stash a freed ANON block; returns true when cached (caller must then + * NOT munmap or un-commit it). */ +static bool direct_cache_put(void* base, size_t map_size) { + size_t budget = direct_cache_budget(); + if (budget == 0) return false; + bool cached = false; + direct_cache_lock(); + if (g_direct_cache_bytes + map_size <= budget) { + for (int i = 0; i < RAY_DIRECT_CACHE_SLOTS; i++) { + if (!g_direct_cache[i].base) { + g_direct_cache[i].base = base; + g_direct_cache[i].map_size = map_size; + g_direct_cache_bytes += map_size; + cached = true; + break; + } + } + } + direct_cache_unlock(); + return cached; +} + +/* Release every cached block back to the kernel (memory pressure). */ +static void direct_cache_drain(void) { + direct_cache_lock(); + for (int i = 0; i < RAY_DIRECT_CACHE_SLOTS; i++) { + if (g_direct_cache[i].base) { + size_t ms = g_direct_cache[i].map_size; + atomic_fetch_sub_explicit(&g_anon_committed, (int64_t)ms, + memory_order_relaxed); + ray_vm_free(g_direct_cache[i].base, ms); + g_direct_cache[i].base = NULL; + g_direct_cache[i].map_size = 0; + g_direct_cache_bytes -= ms; + } + } + direct_cache_unlock(); +} + +void ray_heap_direct_cache_drain(void) { + direct_cache_drain(); +} + /* Direct large allocation: mmap the exact page-rounded size instead of a * power-of-2 oversized buddy pool. Returns a marked ray_t or NULL on failure. */ static ray_t* heap_alloc_direct(ray_heap_t* h, size_t data_size) { @@ -1212,22 +1329,37 @@ static ray_t* heap_alloc_direct(ray_heap_t* h, size_t data_size) { int swap_fd = -1; char* swap_path = NULL; - /* If keeping this in anonymous RAM would push our footprint past the - * watermark (default: physical RAM), back it with a disk file from the - * start so it spills instead of being OOM-killed as its pages fault in. - * (Under lenient overcommit an anon mmap the kernel can't actually back - * SUCCEEDS, then kills us at fault time — so a fallback-on-failure alone - * would not catch it.) Otherwise use anonymous RAM, and fall back to a - * spill file only if the kernel refuses the mapping. */ - bool force_file = heap_anon_would_exceed(map_size); - if (!force_file) - base = ray_vm_alloc(map_size); /* anon RW, page-aligned, counted */ - if (!base) { - base = heap_direct_map_file(h, map_size, &swap_fd, &swap_path); - if (!base) return NULL; - } - if (swap_fd < 0) /* anonymous: counts toward the RAM watermark */ - heap_anon_commit((int64_t)map_size); + /* Reuse cache first: a hit hands back resident, already-committed + * pages — no mmap, no watermark commit (the block never left the + * committed totals while cached), no kernel page-zeroing on fault. */ + size_t cached_size = 0; + base = direct_cache_take(map_size, &cached_size); + if (base) { + map_size = cached_size; + } else { + /* If keeping this in anonymous RAM would push our footprint past the + * watermark (default: physical RAM), back it with a disk file from the + * start so it spills instead of being OOM-killed as its pages fault in. + * (Under lenient overcommit an anon mmap the kernel can't actually back + * SUCCEEDS, then kills us at fault time — so a fallback-on-failure alone + * would not catch it.) Otherwise use anonymous RAM, and fall back to a + * spill file only if the kernel refuses the mapping. + * Under pressure, drain the reuse cache before deciding — its resident + * pages are the first thing to give back. */ + bool force_file = heap_anon_would_exceed(map_size); + if (force_file) { + direct_cache_drain(); + force_file = heap_anon_would_exceed(map_size); + } + if (!force_file) + base = ray_vm_alloc(map_size); /* anon RW, page-aligned, counted */ + if (!base) { + base = heap_direct_map_file(h, map_size, &swap_fd, &swap_path); + if (!base) return NULL; + } + if (swap_fd < 0) /* anonymous: counts toward the RAM watermark */ + heap_anon_commit((int64_t)map_size); + } ray_direct_hdr_t* hdr = (ray_direct_hdr_t*)base; hdr->map_size = map_size; @@ -1442,6 +1574,11 @@ void ray_free(ray_t* v) { ray_sys_track_sub((int64_t)map_size); close(swap_fd); if (swap_path) { unlink(swap_path); ray_sys_free(swap_path); } + } else if (direct_cache_put(base, map_size)) { + /* Stashed for reuse: pages stay resident, so the block keeps + * its committed-RAM and watermark accounting; only the live + * stats above dropped. Eviction (direct_cache_drain) performs + * the deferred un-commit + munmap. */ } else { atomic_fetch_sub_explicit(&g_anon_committed, (int64_t)map_size, memory_order_relaxed); diff --git a/src/mem/heap.h b/src/mem/heap.h index 9d1e0f04..0e82b402 100644 --- a/src/mem/heap.h +++ b/src/mem/heap.h @@ -249,6 +249,11 @@ int64_t ray_heap_anon_peak(void); * to total physical RAM). */ int64_t ray_heap_anon_watermark(void); void ray_heap_set_anon_watermark(int64_t bytes); +/* Release every block held by the direct-allocation reuse cache back to + * the kernel (their committed-RAM accounting drops with them). The cache + * self-drains under watermark pressure; this is for tests and explicit + * memory trimming. */ +void ray_heap_direct_cache_drain(void); /* -------------------------------------------------------------------------- * Pool header: first min-block (64B) of each self-aligned pool. diff --git a/src/ops/agg_engine.c b/src/ops/agg_engine.c index c4c4091d..b4271a43 100644 --- a/src/ops/agg_engine.c +++ b/src/ops/agg_engine.c @@ -1011,7 +1011,8 @@ static ray_t* exec_group_v2_parallel( for (uint32_t a = 0; a < n_aggs; a++) { bool is_list = (vts[a]->out_type == RAY_LIST); - ray_t* out = is_list ? ray_list_new(ng) : ray_vec_new(vts[a]->out_type, ng); + ray_t* out = is_list ? ray_list_new(ng) + : ray_vec_new(vts[a]->out_type, ng); if (!out || RAY_IS_ERR(out)) { agg_table_destroy(>, vts, off, block, n_aggs); agg_desc_free(&d); @@ -1380,7 +1381,8 @@ static ray_t* exec_group_v2_parallel_dense( /* Buffered top_n/bot_n produce a LIST cell per group (a native vector); * median and all streaming aggs produce a scalar out_type cell. */ bool is_list = (vts[a]->out_type == RAY_LIST); - ray_t* out = is_list ? ray_list_new(ng) : ray_vec_new(vts[a]->out_type, ng); + ray_t* out = is_list ? ray_list_new(ng) + : ray_vec_new(vts[a]->out_type, ng); if (!out || RAY_IS_ERR(out)) { agg_dense_slab_destroy_states(gstates, total_slots, vts, off, block, n_aggs); ray_free_raw(occupied_slot); ray_free_raw(first_row_ordered); ray_free_raw(gstates); ray_free_raw(gfirst); @@ -1503,7 +1505,8 @@ static ray_t* exec_group_v2_parallel_radix( ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t nrows, ray_t** key_cols, int64_t* key_syms, const agg_vtable_t** vts, const size_t* off, size_t block, - ray_t* sel, const int64_t* sel_prefix, int64_t n_sel); + ray_t* sel, const int64_t* sel_prefix, int64_t n_sel, + int64_t group_limit); typedef struct { ray_t** key_cols; @@ -1794,10 +1797,14 @@ static uint32_t agg_radix_part_count(uint32_t nworkers, int64_t nrows) { typedef struct { char* buf; uint32_t n, cap; } agg_pay_buf_t; /* n = #records */ /* Reserve room for one more record of `rec` bytes; returns dest ptr or NULL. */ -static char* agg_pay_reserve(agg_pay_buf_t* b, size_t rec) { +static char* agg_pay_reserve(agg_pay_buf_t* b, size_t rec, uint32_t cap0) { if (b->n == b->cap) { if (b->cap > UINT32_MAX / 2) return NULL; - uint32_t nc = b->cap ? b->cap * 2 : 1; + /* First allocation jumps straight to the caller's expected row count + * (uniform-hash estimate). Growing 1->2->4->... instead re-copies the + * whole payload ~2x across tens of thousands of per-(worker,partition) + * buffers — the memmove was 23% of ClickBench q17. */ + uint32_t nc = b->cap ? b->cap * 2 : (cap0 ? cap0 : 1); char* nb = ray_realloc_raw(b->buf, (size_t)nc * rec); if (!nb) return NULL; b->buf = nb; b->cap = nc; @@ -1840,9 +1847,10 @@ static void agg_radix_parts_destroy(agg_radix_part_t* parts, uint32_t nparts, } } -/* Build a result key column of src_col's type for the radix path by un-packing - * the per-group packed key value (column key_idx of each group's representative - * record) from the per-partition `keys` buffers in stable first-seen order. +/* Allocate an EMPTY result key column of src_col's type/width for the radix + * path. agg_key_emit_range below fills it from the per-partition packed `keys` + * buffers in stable first-seen order; the two halves are split so one dispatch + * can fill every key column at once. * * agg_read_key_i64 widened each key into int64 (sign-extend for signed types, * zero-extend for U8/BOOL/SYM intern ids); write_col_i64 is its exact inverse, @@ -1850,29 +1858,61 @@ static void agg_radix_parts_destroy(agg_radix_part_t* parts, uint32_t nparts, * raw memcpy of the original column produced — for SYM it routes through * ray_write_sym at the matching width, and the domain is adopted from src_col * just like agg_gather_key_col. Caller owns the returned column. - * key_idx/n_keys stay uint8_t: both are admission-bounded <=16 (see - * agg_v2_can_handle) and key_idx indexes the packed-key stride below. */ -static ray_t* agg_unpack_key_col(ray_t* src_col, uint32_t key_idx, uint32_t n_keys, - const agg_radix_part_t* parts, uint32_t nparts, - const agg_radix_order_t* pairs, int64_t n) { + * Keys are admitted only as fixed-width integer/temporal/SYM columns + * (agg_v2_can_handle) — there is no STR/variable-width emit path here — but the + * key COUNT is unbounded on the radix strategy (only dense direct-index routing + * caps it at 16), so nothing downstream may size an array by a 16 assumption. */ +static ray_t* agg_unpack_key_col_new(ray_t* src_col, int64_t n) { ray_t* out = col_vec_new(src_col, n); if (!out || RAY_IS_ERR(out)) return out; if (out->type == RAY_SYM) ray_sym_vec_adopt_domain(out, sym_domain_rep(src_col)); out->len = n; - int8_t type = src_col->type; uint8_t attrs = src_col->attrs; - void* dst = ray_data(out); - (void)nparts; - for (int64_t i = 0; i < n; i++) { - uint32_t p = (uint32_t)(pairs[i].idx >> 32); - uint32_t gg = (uint32_t)pairs[i].idx; - write_col_i64(dst, i, - parts[p].keys[(size_t)gg * n_keys + key_idx], type, attrs); - } if (src_col->attrs & RAY_ATTR_HAS_NULLS) out->attrs |= RAY_ATTR_HAS_NULLS; return out; } +/* Un-pack context for the key emit: all n_keys destination columns at once, so + * one dispatch covers the whole key side (mirrors agg_radix_fin_ctx_t, which + * carries every agg's output column through one finalize dispatch). */ +typedef struct { + ray_t* const* key_cols; /* [n_keys] source columns (type/attrs) */ + ray_t* const* outs; /* [n_keys] pre-sized destinations */ + const agg_radix_part_t* parts; + const agg_radix_order_t* pairs; /* [n], stable first-seen order */ + uint32_t n_keys; +} agg_key_emit_ctx_t; + +/* Fill output rows [start,end) of EVERY key column from the packed per-group + * keys. Row i of every column is written by exactly one caller, so a parallel + * dispatch over disjoint [start,end) ranges is race-free: write_col_i64 is a + * pure payload store at index i (down to a 1-byte store for BOOL/U8/SYM_W8 — + * still disjoint, there is no bit-packed representation here) and nothing in + * this loop touches out->attrs or any other shared header field. The nulls + * flag is not derived per row at all: it is copied wholesale from src_col by + * agg_unpack_key_col_new before the dispatch, so no attrs fold is needed. */ +RAY_INLINE void agg_key_emit_range(const agg_key_emit_ctx_t* c, + int64_t start, int64_t end) { + uint32_t n_keys = c->n_keys; + for (uint32_t k = 0; k < n_keys; k++) { + ray_t* out = c->outs[k]; + void* dst = ray_data(out); + int8_t type = c->key_cols[k]->type; + uint8_t attrs = c->key_cols[k]->attrs; + for (int64_t i = start; i < end; i++) { + uint32_t p = (uint32_t)(c->pairs[i].idx >> 32); + uint32_t gg = (uint32_t)c->pairs[i].idx; + write_col_i64(dst, i, + c->parts[p].keys[(size_t)gg * n_keys + k], type, attrs); + } + } +} + +static void agg_key_emit_fn(void* vctx, uint32_t wid, int64_t start, int64_t end) { + (void)wid; + agg_key_emit_range((const agg_key_emit_ctx_t*)vctx, start, end); +} + typedef struct { ray_t** key_cols; const void** key_data; @@ -1885,6 +1925,11 @@ typedef struct { const void** val2_data; const int8_t* val2_types; const bool* val2_hasnull; const uint8_t* val2_esz; uint32_t nw; uint32_t n_parts; + uint32_t part_bits; /* log2(n_parts): hash bits consumed by + * partition selection; phase 2 shifts + * them out before slot indexing */ + uint32_t pay_prime; /* expected rows per (worker,partition) + * buffer — first-allocation capacity */ agg_pay_buf_t* bufs; /* [nw * n_parts] payload records */ agg_radix_part_t* parts; /* [n_parts] */ int phase1_oom; /* set by any Phase-1 worker on push failure */ @@ -1913,6 +1958,19 @@ typedef struct { int64_t* kv_scratch; } agg_radix_ctx_t; +/* Avalanche finalizer shared by the radix scatter (partition selection) and + * phase 2 (partition-local slot index). Both consume DISJOINT bit ranges of + * the same per-row hash — partition from the low log2(n_parts) bits, slot + * from the bits above them — so the raw FNV must be finalized (fmix64, same + * rationale as agg_tuple_hash) and the two sides must compute IDENTICAL + * values for the shift-out split to hold. */ +static inline uint64_t agg_radix_fmix64(uint64_t h) { + h ^= h >> 33; h *= 0xff51afd7ed558ccdULL; + h ^= h >> 33; h *= 0xc4ceb9fe1a85ec53ULL; + h ^= h >> 33; + return h; +} + /* Scatter one ORIGINAL row r's packed payload record into the worker's per- * partition buffer. Returns 0 or -1 on push OOM (sets phase1_oom). */ static inline int agg_radix_scatter_one(agg_radix_ctx_t* c, agg_pay_buf_t* my, @@ -1927,8 +1985,9 @@ static inline int agg_radix_scatter_one(agg_radix_ctx_t* c, agg_pay_buf_t* my, kv[k] = v; h ^= (uint64_t)v; h *= 1099511628211ULL; } + h = agg_radix_fmix64(h); uint32_t p = (uint32_t)(h & (c->n_parts - 1)); - char* rec = agg_pay_reserve(&my[p], c->rec); + char* rec = agg_pay_reserve(&my[p], c->rec, c->pay_prime); if (!rec) { c->phase1_oom = 1; return -1; } int64_t* kdst = (int64_t*)rec; for (uint32_t k = 0; k < n_keys; k++) kdst[k] = kv[k]; @@ -2079,10 +2138,18 @@ static void agg_radix_group_fn(void* vctx, uint32_t wid, int64_t start, int64_t if (gy[a]) { uint8_t ez2 = c->val2_esz[a]; memcpy(gy[a] + (size_t)ri * ez2, rec + c->val2_off[a], ez2); } } - /* Hash the contiguous packed keys (same FNV-1a as the scatter). */ + /* Hash the contiguous packed keys (same FNV-1a as the scatter). + * The scatter consumed the LOW log2(n_parts) bits of this hash + * to pick the partition, so within this partition those bits + * are constant across every record — the slot index must come + * from the bits above them or the open-addressing table + * collapses to 2^(htbits - partbits) usable slots and probing + * degrades to O(ng^2) linear chains (ClickBench q17: 65% of + * the query inside this probe loop at 4096 partitions). */ uint64_t h = 1469598103934665603ULL; for (uint32_t k = 0; k < n_keys; k++) { h ^= (uint64_t)keys[k]; h *= 1099511628211ULL; } - uint64_t slot = h & htmask; + h = agg_radix_fmix64(h); + uint64_t slot = (h >> c->part_bits) & htmask; /* Salt = top 8 bits of the hash (independent of the low-bit slot * index). Compared first on probe to skip ~255/256 full memcmps. */ uint8_t salt = (uint8_t)(h >> 56); @@ -2185,6 +2252,84 @@ static void agg_radix_group_fn(void* vctx, uint32_t wid, int64_t start, int64_t * - LIST out_type (top/bot) cells are NEW ray_t's whose finalize + ray_list_set * COW/retain semantics are not confirmed race-free → LIST aggs are finalized * SERIALLY by the caller (q10 and other scalar shapes get the full win). */ +/* ---- Parallel phase-3 stable ordering ----------------------------------- + * The order map (pairs[input_count], -1-filled) is scattered into by + * partition and then stream-compacted. Serially this is an 80MB touch + + * scatter + full scan per 10M-row query — a flat-scaling wall on high-card + * groups (q17/q18). Parallel version: + * scatter — dispatched over PARTITIONS: each group's first_row is globally + * unique (a row belongs to exactly one group, groups are disjoint across + * partitions), so writes are disjoint and race-free. The serial code's + * per-write duplicate check moves to the aggregate `ordered == ng` check: + * a duplicate (corrupt state) overwrites one entry, the compact count + * comes up short, and the same error path fires. + * compact — two passes over fixed chunks: per-chunk non-empty counts, a + * serial prefix over the (few hundred) chunk counts, then in-place writes + * at each chunk's prefix offset. In-place is forward-safe: chunk k's + * write region [prefix[k], prefix[k]+cnt[k]) never overlaps a LATER + * chunk's unread region (prefix[j+1] <= (j+1)*C <= k*C for j < k), and + * within a chunk dst <= src with ascending iteration. */ +typedef struct { + agg_radix_part_t* parts; + agg_radix_order_t* pairs; + int64_t input_count; + _Atomic(int) fail; +} agg_ord_scatter_ctx_t; + +static void agg_ord_scatter_fn(void* vctx, uint32_t wid, int64_t start, + int64_t end) { + (void)wid; + agg_ord_scatter_ctx_t* c = (agg_ord_scatter_ctx_t*)vctx; + if (atomic_load_explicit(&c->fail, memory_order_relaxed)) return; + for (int64_t p = start; p < end; p++) { + agg_radix_part_t* pr = &c->parts[p]; + for (int64_t gg = 0; gg < pr->ng; gg++) { + int64_t first = pr->first_row[gg]; + if (first < 0 || first >= c->input_count) { + atomic_store_explicit(&c->fail, 1, memory_order_relaxed); + return; + } + c->pairs[first].idx = ((int64_t)p << 32) | (uint32_t)gg; + } + } +} + +typedef struct { + agg_radix_order_t* pairs; + int64_t input_count; + int64_t chunk; /* elements per chunk */ + int64_t* counts; /* [n_chunks]: pass A out / pass B prefix in */ +} agg_ord_compact_ctx_t; + +static void agg_ord_count_fn(void* vctx, uint32_t wid, int64_t start, + int64_t end) { + (void)wid; + agg_ord_compact_ctx_t* c = (agg_ord_compact_ctx_t*)vctx; + for (int64_t ch = start; ch < end; ch++) { + int64_t lo = ch * c->chunk; + int64_t hi = lo + c->chunk; + if (hi > c->input_count) hi = c->input_count; + int64_t n = 0; + for (int64_t i = lo; i < hi; i++) + n += (c->pairs[i].idx != -1); + c->counts[ch] = n; + } +} + +static void agg_ord_compact_fn(void* vctx, uint32_t wid, int64_t start, + int64_t end) { + (void)wid; + agg_ord_compact_ctx_t* c = (agg_ord_compact_ctx_t*)vctx; + for (int64_t ch = start; ch < end; ch++) { + int64_t lo = ch * c->chunk; + int64_t hi = lo + c->chunk; + if (hi > c->input_count) hi = c->input_count; + int64_t w = c->counts[ch]; /* prefix offset for this chunk */ + for (int64_t i = lo; i < hi; i++) + if (c->pairs[i].idx != -1) c->pairs[w++].idx = c->pairs[i].idx; + } +} + typedef struct { agg_radix_part_t* parts; const agg_radix_order_t* pairs; /* [ng], stable first-seen order */ @@ -2233,11 +2378,96 @@ static void agg_radix_finalize_fn(void* vctx, uint32_t wid, int64_t start, int64 } } +/* Bounded first-seen selection for the HEAD(GROUP) limit hint: pick the + * `n_emit` groups with the SMALLEST first_row across all partitions — which + * ARE the first `n_emit` groups in first-seen order — and return them as the + * usual (part << 32 | local gid) order array, ascending by first_row. + * + * Kept noinline so the (cold, hint-only) selection code neither inflates nor + * perturbs the register allocation of exec_group_v2_parallel_radix's hot body + * (mirrors group.c's noinline isolation of its per-partition path). + * `*rc`: 0 ok, 1 oom, 2 corrupt first_row (caller raises the order error). */ +static agg_radix_order_t* __attribute__((noinline)) +agg_radix_select_first_n(const agg_radix_part_t* parts, uint32_t n_parts, + int64_t input_count, int64_t n_emit, int* rc) { + *rc = 0; + agg_radix_order_t* sel_pairs = + ray_alloc_raw((size_t)n_emit * sizeof(agg_radix_order_t)); + ray_t* hk_hdr = NULL; + int64_t* hkey = sel_pairs + ? (int64_t*)scratch_alloc(&hk_hdr, (size_t)n_emit * sizeof(int64_t)) + : NULL; + if (!sel_pairs || !hkey) { + ray_free_raw(sel_pairs); scratch_free(hk_hdr); + *rc = 1; return NULL; + } + /* Max-heap over first_row keeping the n_emit smallest. */ + int64_t hn = 0; + bool ok = true; + for (uint32_t p = 0; p < n_parts && ok; p++) { + const int64_t* fr = parts[p].first_row; + for (int64_t gg = 0; gg < parts[p].ng; gg++) { + int64_t first = fr[gg]; + if (first < 0 || first >= input_count) { ok = false; break; } + int64_t payload = ((int64_t)p << 32) | (uint32_t)gg; + if (hn < n_emit) { + int64_t i = hn++; + hkey[i] = first; sel_pairs[i].idx = payload; + while (i > 0) { /* sift up */ + int64_t par = (i - 1) / 2; + if (hkey[par] >= hkey[i]) break; + int64_t tk = hkey[par]; hkey[par] = hkey[i]; hkey[i] = tk; + int64_t tp = sel_pairs[par].idx; + sel_pairs[par].idx = sel_pairs[i].idx; sel_pairs[i].idx = tp; + i = par; + } + } else if (first < hkey[0]) { + hkey[0] = first; sel_pairs[0].idx = payload; + int64_t i = 0; + for (;;) { /* sift down */ + int64_t l = 2 * i + 1, r = l + 1, m = i; + if (l < n_emit && hkey[l] > hkey[m]) m = l; + if (r < n_emit && hkey[r] > hkey[m]) m = r; + if (m == i) break; + int64_t tk = hkey[m]; hkey[m] = hkey[i]; hkey[i] = tk; + int64_t tp = sel_pairs[m].idx; + sel_pairs[m].idx = sel_pairs[i].idx; sel_pairs[i].idx = tp; + i = m; + } + } + } + } + /* Heap-sort the selected entries into ascending first_row order. */ + if (ok) { + for (int64_t end = hn - 1; end > 0; end--) { + int64_t tk = hkey[0]; hkey[0] = hkey[end]; hkey[end] = tk; + int64_t tp = sel_pairs[0].idx; + sel_pairs[0].idx = sel_pairs[end].idx; sel_pairs[end].idx = tp; + int64_t i = 0; + for (;;) { + int64_t l = 2 * i + 1, r = l + 1, m = i; + if (l < end && hkey[l] > hkey[m]) m = l; + if (r < end && hkey[r] > hkey[m]) m = r; + if (m == i) break; + tk = hkey[m]; hkey[m] = hkey[i]; hkey[i] = tk; + tp = sel_pairs[m].idx; + sel_pairs[m].idx = sel_pairs[i].idx; sel_pairs[i].idx = tp; + i = m; + } + } + ok = hn == n_emit; + } + scratch_free(hk_hdr); + if (!ok) { ray_free_raw(sel_pairs); *rc = 2; return NULL; } + return sel_pairs; +} + static ray_t* exec_group_v2_parallel_radix( ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t nrows, ray_t** key_cols, int64_t* key_syms, const agg_vtable_t** vts, const size_t* off, size_t block, - ray_t* sel, const int64_t* sel_prefix, int64_t n_sel) { + ray_t* sel, const int64_t* sel_prefix, int64_t n_sel, + int64_t group_limit) { ray_op_ext_t* ext = find_ext(g, op->id); uint32_t n_keys = ext->n_keys, n_aggs = ext->n_aggs; ray_pool_t* pool = ray_pool_get(); @@ -2292,6 +2522,11 @@ static ray_t* exec_group_v2_parallel_radix( .val_data = val_data, .val_types = val_types, .val_hasnull = val_hasnull, .val_esz = val_esz, .val2_data = val2_data, .val2_types = val2_types, .val2_hasnull = val2_hasnull, .val2_esz = val2_esz, .nw = nw, .n_parts = n_parts, + .part_bits = (uint32_t)__builtin_ctz(n_parts), + /* Uniform-hash expectation with 25% slack; ≥8 so tiny buffers don't + * immediately re-double. */ + .pay_prime = (uint32_t)((uint64_t)(sel ? n_sel : nrows) + / ((uint64_t)nw * n_parts) * 5 / 4 + 8), .bufs = bufs, .parts = parts, .phase1_oom = 0, .rec = rec, .row_off = row_off, .needs_row = needs_row, .sel = sel, .sel_prefix = sel_prefix, @@ -2331,7 +2566,39 @@ static ray_t* exec_group_v2_parallel_radix( * compact those occupied positions in one linear pass. This restores * first-seen order without a comparison sort or a routing threshold. */ int64_t input_count = sel ? n_sel : nrows; - agg_radix_order_t* pairs = ray_alloc_raw( + + /* Bounded emit under a HEAD(GROUP) limit hint: when only the first + * `group_limit` groups are wanted, selecting them directly is O(ng) with a + * `group_limit`-sized heap, versus the full path's O(input_count) order map + * (an 80MB alloc + memset + scatter + compact on a 10M-row input) followed + * by a full key-unpack and finalize of every group. The N groups with the + * SMALLEST first_row ARE the first N groups in first-seen order, so the + * emitted prefix is byte-identical to trimming the full result to N. */ + /* Bounded emit under a HEAD(GROUP) limit hint: when only the first + * `group_limit` groups are wanted, selecting them directly is O(ng) with a + * `group_limit`-sized heap, versus the full path's O(input_count) order map + * (an 80MB alloc + memset + scatter + compact on a 10M-row input) followed + * by a full key-unpack and finalize of every group. The N groups with the + * SMALLEST first_row ARE the first N groups in first-seen order, so the + * emitted prefix is byte-identical to trimming the full result to N. */ + int64_t n_emit = ng; + agg_radix_order_t* pairs = NULL; + if (group_limit > 0 && ng > group_limit) { + int rc = 0; + n_emit = group_limit; + agg_radix_order_t* sel_pairs = agg_radix_select_first_n( + parts, n_parts, input_count, n_emit, &rc); + if (!sel_pairs) { + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); + for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); + ray_free_raw(bufs); ray_free_raw(parts); + agg_desc_free(&d); + return rc == 1 ? ray_error("oom", NULL) + : ray_error("group", "failed to order radix groups"); + } + pairs = sel_pairs; + } else { + pairs = ray_alloc_raw( (size_t)(input_count > 0 ? input_count : 1) * sizeof(agg_radix_order_t)); if (!pairs) { agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); @@ -2340,23 +2607,66 @@ static ray_t* exec_group_v2_parallel_radix( agg_desc_free(&d); return ray_error("oom", NULL); } - for (int64_t i = 0; i < input_count; i++) pairs[i].idx = -1; + /* -1 fill as bytes: 0xFF.. == -1 for int64, and memset vectorizes — + * this is an 80MB serial touch on a 10M-row input, worth the idiom. */ + memset(pairs, 0xFF, + (size_t)(input_count > 0 ? input_count : 1) * sizeof(agg_radix_order_t)); bool order_ok = true; - for (uint32_t p = 0; p < n_parts && order_ok; p++) { - for (int64_t gg = 0; gg < parts[p].ng; gg++) { - int64_t first = parts[p].first_row[gg]; - if (first < 0 || first >= input_count || pairs[first].idx != -1) { + int64_t ordered = 0; + bool ord_parallel_done = false; + if (pool && nw > 1 && input_count >= (1 << 20)) { + const int64_t ORD_CHUNK = 1 << 17; + int64_t n_chunks = (input_count + ORD_CHUNK - 1) / ORD_CHUNK; + ray_t* ordcnt_hdr = NULL; + int64_t* ord_counts = (int64_t*)scratch_alloc(&ordcnt_hdr, + (size_t)n_chunks * sizeof(int64_t)); + if (ord_counts) { + agg_ord_scatter_ctx_t sctx = { + .parts = parts, .pairs = pairs, + .input_count = input_count, .fail = 0, + }; + ray_pool_dispatch(pool, agg_ord_scatter_fn, &sctx, + (int64_t)n_parts); + if (!atomic_load_explicit(&sctx.fail, memory_order_relaxed)) { + agg_ord_compact_ctx_t cctx = { + .pairs = pairs, .input_count = input_count, + .chunk = ORD_CHUNK, .counts = ord_counts, + }; + ray_pool_dispatch(pool, agg_ord_count_fn, &cctx, n_chunks); + /* Exclusive prefix (serial over a few hundred chunks). */ + int64_t run = 0; + for (int64_t ch = 0; ch < n_chunks; ch++) { + int64_t n = ord_counts[ch]; + ord_counts[ch] = run; + run += n; + } + ray_pool_dispatch(pool, agg_ord_compact_fn, &cctx, n_chunks); + ordered = run; + order_ok = ordered == ng; + ord_parallel_done = true; + } else { order_ok = false; - break; + ord_parallel_done = true; /* bounds violation → error path */ } - pairs[first].idx = ((int64_t)p << 32) | (uint32_t)gg; + scratch_free(ordcnt_hdr); } } - int64_t ordered = 0; - if (order_ok) { - for (int64_t i = 0; i < input_count; i++) - if (pairs[i].idx != -1) pairs[ordered++].idx = pairs[i].idx; - order_ok = ordered == ng; + if (!ord_parallel_done) { + for (uint32_t p = 0; p < n_parts && order_ok; p++) { + for (int64_t gg = 0; gg < parts[p].ng; gg++) { + int64_t first = parts[p].first_row[gg]; + if (first < 0 || first >= input_count || pairs[first].idx != -1) { + order_ok = false; + break; + } + pairs[first].idx = ((int64_t)p << 32) | (uint32_t)gg; + } + } + if (order_ok) { + for (int64_t i = 0; i < input_count; i++) + if (pairs[i].idx != -1) pairs[ordered++].idx = pairs[i].idx; + order_ok = ordered == ng; + } } if (!order_ok) { ray_free_raw(pairs); @@ -2366,6 +2676,7 @@ static ray_t* exec_group_v2_parallel_radix( agg_desc_free(&d); return ray_error("group", "failed to order radix groups"); } + } /* end full-order path */ ray_t* result = ray_table_new(n_keys + n_aggs); if (!result || RAY_IS_ERR(result)) { @@ -2380,21 +2691,58 @@ static ray_t* exec_group_v2_parallel_radix( /* Emit key columns by sequential un-pack from the contiguous per-partition * packed-key buffers (cache-friendly), NOT a scattered gather of the * original columns at first_row[]. Byte-identical to agg_gather_key_col - * (see agg_unpack_key_col), incl. SYM payload + domain. */ - for (uint32_t k = 0; k < n_keys; k++) { - ray_t* kc = agg_unpack_key_col(key_cols[k], k, n_keys, parts, n_parts, - pairs, ng); - if (!kc || RAY_IS_ERR(kc)) { - ray_free_raw(pairs); - agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); - for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); - ray_free_raw(bufs); ray_free_raw(parts); - agg_desc_free(&d); - ray_release(result); return kc ? kc : ray_error("oom", NULL); + * (see agg_unpack_key_col_new), incl. SYM payload + domain. + * + * All n_keys destinations are built BEFORE the fill so one dispatch covers + * the whole key side; they enter the table only once every row is written. + * n_keys is NOT bounded to 16 on this path (only dense direct-index routing + * caps it, in agg_dense_plan), so the handle array is a scratch carve, not + * a stack array — ASan caught the stack version overflowing on a 17-key + * group-by (test/rfl/group/radix_key_emit_parallel.rfl). */ + ray_t* kouts_hdr = NULL; + ray_t** kouts = (ray_t**)scratch_calloc(&kouts_hdr, (size_t)n_keys * sizeof(ray_t*)); + ray_t* kerr = kouts ? NULL : ray_error("oom", NULL); + uint32_t k_built = 0; + for (; !kerr && k_built < n_keys; k_built++) { + ray_t* kc = agg_unpack_key_col_new(key_cols[k_built], n_emit); + if (!kc || RAY_IS_ERR(kc)) { kerr = kc ? kc : ray_error("oom", NULL); break; } + kouts[k_built] = kc; + } + if (!kerr) { + agg_key_emit_ctx_t kctx = { + .key_cols = key_cols, .outs = kouts, .parts = parts, + .pairs = pairs, .n_keys = n_keys, + }; + /* Parallelize the un-pack over the OUTPUT rows on the same terms as the + * phase-3 finalize below: worthwhile only when ngroups is large. Every + * bounded-emit shape (the HEAD(GROUP) limit hint — q17 emits 10 rows) + * and every low-cardinality group-by stays on the identical serial + * range call, so there is no dispatch overhead where there is no win. */ + if (ray_pool_par_dispatch_ok(pool, n_emit, RAY_PARALLEL_THRESHOLD)) { + ray_pool_dispatch(pool, agg_key_emit_fn, &kctx, n_emit); + /* A cancelled dispatch drains its tickets WITHOUT running fn, so the + * key columns can be left partly uninitialized (for SYM that would + * be an out-of-domain id). Never hand that back — bail. */ + if (pool_cancelled(pool)) kerr = ray_error("cancel", NULL); + } else { + agg_key_emit_range(&kctx, 0, n_emit); } - result = ray_table_add_col(result, key_syms[k], kc); - ray_release(kc); } + if (kerr) { + for (uint32_t k = 0; k < k_built; k++) ray_release(kouts[k]); + scratch_free(kouts_hdr); + ray_free_raw(pairs); + agg_radix_parts_destroy(parts, n_parts, vts, off, block, n_aggs); + for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); + ray_free_raw(bufs); ray_free_raw(parts); + agg_desc_free(&d); + ray_release(result); return kerr; + } + for (uint32_t k = 0; k < n_keys; k++) { + result = ray_table_add_col(result, key_syms[k], kouts[k]); + ray_release(kouts[k]); + } + scratch_free(kouts_hdr); /* Pre-allocate every agg's output column up front so the parallel finalize * pass can write disjoint slices into all scalar columns at once. LIST @@ -2415,7 +2763,8 @@ static ray_t* exec_group_v2_parallel_radix( /* Buffered top_n/bot_n produce a LIST cell per group (a native vector); * median and all streaming aggs produce a scalar out_type cell. */ bool is_list = (vts[a]->out_type == RAY_LIST); - ray_t* out = is_list ? ray_list_new(ng) : ray_vec_new(vts[a]->out_type, ng); + ray_t* out = is_list ? ray_list_new(n_emit) + : ray_vec_new(vts[a]->out_type, n_emit); if (!out || RAY_IS_ERR(out)) { for (uint32_t b = 0; b < a; b++) ray_release(outs[b]); ray_free_raw(pairs); @@ -2425,7 +2774,7 @@ static ray_t* exec_group_v2_parallel_radix( scratch_free(outs_hdr); agg_desc_free(&d); ray_release(result); return out ? out : ray_error("oom", NULL); } - out->len = ng; + out->len = n_emit; outs[a] = out; kparams[a] = (ext->agg_k ? ext->agg_k[a] : 0); } @@ -2433,7 +2782,7 @@ static ray_t* exec_group_v2_parallel_radix( /* Parallelize the scalar finalize over the output rows when ngroups is large * (the q10/high-card win). Small ng (incl. every low-card shape) keeps the * trivial serial loop → zero dispatch overhead, no regression. */ - bool par_fin = (pool && ng >= RAY_PARALLEL_THRESHOLD); + bool par_fin = (pool && n_emit >= RAY_PARALLEL_THRESHOLD); if (par_fin) { uint8_t* saw_null = ray_calloc_raw((size_t)((size_t)nw * (n_aggs ? n_aggs : 1)) * (1)); ray_t* scalar_hdr = NULL; @@ -2448,7 +2797,7 @@ static ray_t* exec_group_v2_parallel_radix( .block = block, .n_aggs = n_aggs, .agg_k = kparams, .outs = scalar_outs, .saw_null = saw_null, }; - ray_pool_dispatch(pool, agg_radix_finalize_fn, &fc, ng); + ray_pool_dispatch(pool, agg_radix_finalize_fn, &fc, n_emit); /* OR the deferred HAS_NULLS flag once, serially. */ for (uint32_t a = 0; a < n_aggs; a++) { if (vts[a]->out_type == RAY_LIST) continue; @@ -2469,7 +2818,7 @@ static ray_t* exec_group_v2_parallel_radix( /* Serial finalize: LIST aggs always (ray_list_set COW/retain not * confirmed race-safe), and all aggs when the parallel pass was skipped. */ if (is_list || !par_fin) { - for (int64_t i = 0; i < ng; i++) { + for (int64_t i = 0; i < n_emit; i++) { uint32_t p = (uint32_t)(pairs[i].idx >> 32); uint32_t gg = (uint32_t)(pairs[i].idx & 0xffffffffu); ray_t* cell = vts[a]->finalize(parts[p].states + (size_t)gg * block + off[a], NULL, kparams[a]); @@ -2521,7 +2870,8 @@ static ray_t* agg_build_compact(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * compact fallback the design permits for the non-chunked shapes. */ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t nrows, ray_t* sel, - const int64_t* sel_prefix, int64_t n_sel) { + const int64_t* sel_prefix, int64_t n_sel, + int64_t group_limit) { ray_op_ext_t* ext = find_ext(g, op->id); /* Exact-size carve for the per-key column pointers + syms (one block, both @@ -2568,7 +2918,8 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, ray_release(idxb); \ return compact ? compact : ray_error("oom", NULL); \ } \ - ray_t* r = exec_group_v2_run(g, op, compact, n_sel, NULL, NULL, 0); \ + ray_t* r = exec_group_v2_run(g, op, compact, n_sel, NULL, NULL, 0, \ + group_limit); \ ray_release(compact); ray_release(idxb); \ return r; \ } while (0) @@ -2620,7 +2971,8 @@ static ray_t* exec_group_v2_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, /* Sparse integer/SYM ranges use radix deterministically. No sampled * cardinality or cache-size crossover is baked into routing. */ ray_t* r = exec_group_v2_parallel_radix(g, op, tbl, nrows, - key_cols, key_syms, vts, off, block, sel, sel_prefix, n_sel); + key_cols, key_syms, vts, off, block, sel, sel_prefix, n_sel, + group_limit); agg_vo_free(&vo); scratch_free(kc_hdr); return r; } /* Hash fallback (F64 / STR keys): not a chunked strategy — compact. */ @@ -2763,9 +3115,11 @@ static ray_t* agg_build_compact(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * with sel=NULL). Representative (first_row) indices stay in ORIGINAL-row space * throughout so result key columns gather correctly (SYM domains preserved); the * output order is unspecified per the v2 contract. */ -ray_t* exec_group_v2(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { +ray_t* exec_group_v2(ray_graph_t* g, ray_op_t* op, ray_t* tbl, + int64_t group_limit) { if (!g || !g->selection) - return exec_group_v2_run(g, op, tbl, ray_table_nrows(tbl), NULL, NULL, 0); + return exec_group_v2_run(g, op, tbl, ray_table_nrows(tbl), NULL, NULL, 0, + group_limit); int64_t src_nrows = ray_table_nrows(tbl); ray_rowsel_t* sm = ray_rowsel_meta(g->selection); @@ -2773,7 +3127,7 @@ ray_t* exec_group_v2(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { * applied here — fall back to the unfiltered run (matches the scalar-agg * guard in group.c, which also only honors a selection when nrows match). */ if (sm->nrows != src_nrows) - return exec_group_v2_run(g, op, tbl, src_nrows, NULL, NULL, 0); + return exec_group_v2_run(g, op, tbl, src_nrows, NULL, NULL, 0, group_limit); int64_t n_sel = sm->total_pass; ray_t* prefix_block = agg_sel_build_prefix(g->selection); @@ -2784,7 +3138,8 @@ ray_t* exec_group_v2(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { * apply it; the rowsel is passed explicitly via the sel argument instead. */ ray_t* saved_sel = g->selection; g->selection = NULL; - ray_t* result = exec_group_v2_run(g, op, tbl, src_nrows, saved_sel, sel_prefix, n_sel); + ray_t* result = exec_group_v2_run(g, op, tbl, src_nrows, saved_sel, sel_prefix, + n_sel, group_limit); g->selection = saved_sel; ray_release(prefix_block); diff --git a/src/ops/agg_engine.h b/src/ops/agg_engine.h index 2d66c877..102c87a5 100644 --- a/src/ops/agg_engine.h +++ b/src/ops/agg_engine.h @@ -14,8 +14,12 @@ extern bool ray_agg_engine_v2; * Conservative: any uncertainty → false → caller uses the existing engine. */ bool agg_v2_can_handle(ray_graph_t* g, ray_op_t* op, ray_t* tbl); -/* Precondition: agg_v2_can_handle(g, op, tbl) returned true. */ -ray_t* exec_group_v2(ray_graph_t* g, ray_op_t* op, ray_t* tbl); +/* Precondition: agg_v2_can_handle(g, op, tbl) returned true. + * `group_limit` is the HEAD(GROUP) row-limit HINT (0 = no limit): when + * positive, an engine strategy may emit only the first `group_limit` groups in + * first-seen order. Advisory only — the caller trims the result regardless. */ +ray_t* exec_group_v2(ray_graph_t* g, ray_op_t* op, ray_t* tbl, + int64_t group_limit); /* Dense group assignment over the group's key columns, first-occurrence * order (key count unbounded since cut 3; the dense planner self-limits). */ diff --git a/src/ops/arith.c b/src/ops/arith.c index 1bd67f2f..9be671ef 100644 --- a/src/ops/arith.c +++ b/src/ops/arith.c @@ -286,13 +286,29 @@ ray_t* ray_idiv_fn(ray_t* a, ray_t* b) { ray_type_name(a->type), ray_type_name(b->type)); if (RAY_ATOM_IS_NULL(a) || RAY_ATOM_IS_NULL(b)) return ray_typed_null(-RAY_I64); - double bv = as_f64(b); - if (bv == 0.0) - return ray_typed_null(-RAY_I64); - double q = floor(as_f64(a) / bv); - if (q < (double)INT64_MIN || q > (double)INT64_MAX) + if (is_float_op(a, b)) { + double bv = as_f64(b); + if (bv == 0.0) + return ray_typed_null(-RAY_I64); + double q = floor(as_f64(a) / bv); + if (q >= 0x1p63 /* 2^63: first double past INT64_MAX */ || q < (double)INT64_MIN) + return ray_typed_null(-RAY_I64); + return make_i64((int64_t)q); + } + /* Integer div: stay in int64 space. The double round-trip above silently + * loses precision for magnitudes > 2^53 and is UB for q == 2^63. */ + int64_t bv = as_i64(b); + int64_t la = as_i64(a); + /* bv==0 → null; INT64_MIN/-1 overflow → null (unreachable while INT64_MIN + * is the i64 null sentinel and caught above, but keep the guard so the + * scalar path can never trip signed-overflow UB — symmetric with the + * vector floor_idiv_i64_checked kernel). */ + if (bv == 0 || (la == INT64_MIN && bv == -1)) return ray_typed_null(-RAY_I64); - return make_i64((int64_t)q); + int64_t q = la / bv; + if (la % bv != 0 && ((la < 0) != (bv < 0))) + q--; /* floor toward -inf */ + return make_i64(q); } ray_t* ray_mod_fn(ray_t* a, ray_t* b) { diff --git a/src/ops/cdfuse.c b/src/ops/cdfuse.c new file mode 100644 index 00000000..b602e873 --- /dev/null +++ b/src/ops/cdfuse.c @@ -0,0 +1,658 @@ +/* + * Copyright (c) 2025-2026 Anton Kundenko + * All rights reserved. + + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +/* Fused grouped count-distinct (spec Part B). Single pass over rows: + * phase 1 scatters compact [pairhash][k][v][row] records into per-(worker, + * partition) buffers; phase 2 walks each partition once with two + * partition-local open-addressing tables — a (k,v) dedupe table and a k + * table — producing PARTIAL per-key (distinct count, first_row) laid out in + * key-hash buckets; phase 3 dispatches one task per bucket to merge those + * partials into global totals, which are then ordered and emitted as + * (k, distinct, first_row). No intermediate pairs table, no second group + * pipeline. + * + * Partitioning is by the PAIR hash fmix64(hash(k) ^ hash(v)), NOT by the + * key hash. Key-hash partitioning makes each key's ENTIRE row set land in + * one partition, so a single skewed key (e.g. a region owning half the + * table) serializes phase 2 behind one core — measured 35% SLOWER than the + * unfused path on a 100M-row / 9K-key / heavy-skew query. The pair hash is + * uniform regardless of key skew, at the price of every partition seeing + * (nearly) every key — hence the additive merge in phase 3 and the 1024 + * partition cap that bounds the merge input. + * + * The dedupe table's slot index uses pair-hash bits ABOVE the partition + * bits (the agg_engine hash-bit-overlap lesson); the k table indexes on an + * independent hash of k alone, which takes no part in partition selection. */ + +#include +#include +#include +#include "rayforce.h" +#include "core/pool.h" +#include "mem/heap.h" /* ray_heap_anon_watermark */ +#include "ops/internal.h" /* scratch_*, read_col_i64 */ +#include "ops/hash.h" /* ray_hash_i64 */ +#include "ops/cdfuse.h" +#include "table/sym.h" /* RAY_IS_SYM */ + +/* Peak-footprint estimate used by the admission gate below: 32B/row for + * phase 1's records plus up to 24B per (partition, key) pair in phase 2, + * itself bounded by the row count — see the KNOWN MEMORY BOUND comment at + * the phase-3 call site. 56B/row is that worst case (all-distinct input), + * both phases' buffers alive simultaneously until cdf_free_all runs. */ +#define CDF_BYTES_PER_ROW 56 + +/* ══════════════════════════════════════════ + * Shared helpers + * ══════════════════════════════════════════ */ + +/* Avalanche finalizer — identical body to agg_radix_fmix64 (agg_engine.c, + * static there). Partition selection consumes the LOW log2(n_parts) bits of + * the pair hash and the dedupe slot index the bits ABOVE them, so the raw + * hash must be finalized for the shift-out split to hold. */ +static inline uint64_t cdf_fmix64(uint64_t h) { + h ^= h >> 33; + h *= 0xff51afd7ed558ccdULL; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53ULL; + h ^= h >> 33; + return h; +} + +/* Growable per-(worker, partition) record buffer. */ +typedef struct { + char* buf; + uint32_t n, cap; +} cdf_buf_t; + +#define CDF_REC 32 /* [pairhash 8][k 8][v 8][row 8] */ + +/* Upper bound on partitions. Phase 3's merge input is + * n_parts × distinct-keys-per-partition and pair-hash partitioning lets every + * partition see nearly every key, so the cap is what keeps the merge bounded; + * 1024 still gives ample parallelism (and equals RAY_POOL_INIT_TASKS, so + * ray_pool_dispatch_n never needs to grow its ring). */ +#define CDF_MAX_PARTS 1024u + +/* Reserve room for one more record; returns dest ptr or NULL on OOM. */ +static char* cdf_reserve(cdf_buf_t* b, uint32_t prime) { + if (b->n == b->cap) { + if (b->cap > UINT32_MAX / 2) return NULL; + uint32_t nc = b->cap ? b->cap * 2 : (prime ? prime : 64); + char* nb = (char*)ray_realloc_raw(b->buf, (size_t)nc * CDF_REC); + if (!nb) return NULL; + b->buf = nb; + b->cap = nc; + } + return b->buf + (size_t)b->n++ * CDF_REC; +} + +/* ══════════════════════════════════════════ + * Phase 1 — scatter + * ══════════════════════════════════════════ */ + +typedef struct { + const void* kdata; + int8_t ktype; + uint8_t kattrs; + const void* vdata; + int8_t vtype; + uint8_t vattrs; + uint32_t n_parts, nw; + cdf_buf_t* bufs; /* [nw * n_parts] */ + uint32_t prime; /* first-allocation capacity per buf */ + _Atomic(int) oom; + /* Rows actually scattered. ray_pool_dispatch clamps its task count to + * the ring capacity but RECOMPUTES the grain, so no row is dropped by + * clamping — however a cancelled pool (pool->cancelled) skips claimed + * tasks outright. Comparing this against nrows turns any such silent + * row loss into a NULL decline instead of a short answer. */ + _Atomic(int64_t) rows_done; +} cdf_p1_ctx_t; + +static void cdf_p1_fn(void* vctx, uint32_t wid, int64_t start, int64_t end) { + cdf_p1_ctx_t* c = (cdf_p1_ctx_t*)vctx; + if (atomic_load_explicit(&c->oom, memory_order_relaxed)) return; + cdf_buf_t* my = &c->bufs[(size_t)(wid % c->nw) * c->n_parts]; + for (int64_t r = start; r < end; r++) { + int64_t k = read_col_i64(c->kdata, r, c->ktype, c->kattrs); + int64_t v = read_col_i64(c->vdata, r, c->vtype, c->vattrs); + /* PAIR hash: uniform even when one key owns most of the table. + * The odd-multiplier on the k side is LOAD-BEARING, not decoration: a + * bare `hash(k) ^ hash(v)` cancels to 0 for every row where k == v, so + * a `(count (distinct k)) by: k` — or any strongly correlated column + * pair — would pile every row into partition 0 at dedupe slot 0 and run + * one giant serial probe cluster (measured: 2m42s on 20M rows vs 0.09s + * uncorrelated). Multiplying one side by the golden-ratio constant + * makes the combine asymmetric, so k == v hashes like any other pair. */ + uint64_t h = cdf_fmix64(ray_hash_i64(k) * 0x9E3779B97F4A7C15ULL ^ + ray_hash_i64(v)); + uint32_t p = (uint32_t)(h & (c->n_parts - 1)); + char* rec = cdf_reserve(&my[p], c->prime); + if (!rec) { + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + return; + } + ((uint64_t*)rec)[0] = h; + ((int64_t*)rec)[1] = k; + ((int64_t*)rec)[2] = v; + ((int64_t*)rec)[3] = r; + } + atomic_fetch_add_explicit(&c->rows_done, end - start, memory_order_relaxed); +} + +/* ══════════════════════════════════════════ + * Phase 2 — per-partition dedupe + count + * ══════════════════════════════════════════ */ + +/* One group's PARTIAL result inside one partition: this partition's share of + * the key's distinct-value count and the minimum row index it saw for the key. + * Phase 3 sums the counts and takes the MIN of the firsts. Kept as a + * struct-of-three (not three parallel slabs pre-sized to the partition's + * record count) so the output memory tracks the group count instead of the row + * count — a 100M-row / 1K-group run pays 24 bytes per GROUP, not per ROW. */ +typedef struct { + int64_t key, cnt, first; +} cdf_grp_t; + +/* Phase-3 merge buckets. Phase 2 lays its partial triples out grouped by + * merge bucket so phase 3 can dispatch one task per bucket: bucket m walks + * bucket m of EVERY partition, and because the bucket is a function of the key + * alone, keys are disjoint across buckets — no cross-task coordination. */ +#define CDF_MERGE_PARTS 64 + +/* Merge bucket of a key, taken from bits 32..37 of fmix64(hash(k)). The + * phase-2 k table and the phase-3 per-bucket table both index on the LOW bits + * of that same value, so the bucket selector is taken from the high half to + * keep the two selectors independent (the partition itself came from the pair + * hash, which shares no bits with this one). */ +static inline uint32_t cdf_bucket(uint64_t kh) { + return (uint32_t)((kh >> 32) & (CDF_MERGE_PARTS - 1)); +} + +typedef struct { + cdf_grp_t* g; /* ng partial triples, laid out bucket by bucket */ + int64_t ng; + int64_t boff[CDF_MERGE_PARTS + 1]; /* bucket m is g[boff[m], boff[m+1]) */ +} cdf_part_t; + +typedef struct { + cdf_buf_t* bufs; + cdf_part_t* parts; + uint32_t nw, n_parts, part_bits; + _Atomic(int) oom; + /* Partitions actually processed. ray_pool_dispatch_n CLAMPS its task + * count to the ring capacity when the growth realloc fails (see + * RAY_POOL_INIT_TASKS in core/pool.h) and a cancelled pool skips claimed + * tasks — either way the tail partitions never run, keep ng == 0, and + * their groups would silently vanish from the result. The wrapper + * compares this against n_parts and declines (NULL) on mismatch. */ + _Atomic(int64_t) done; +} cdf_p2_ctx_t; + +static void cdf_p2_fn(void* vctx, uint32_t wid, int64_t start, int64_t end) { + (void)wid; + cdf_p2_ctx_t* c = (cdf_p2_ctx_t*)vctx; + if (atomic_load_explicit(&c->oom, memory_order_relaxed)) return; + for (int64_t p = start; p < end; p++) { + int64_t total = 0; + for (uint32_t w = 0; w < c->nw; w++) + total += c->bufs[(size_t)w * c->n_parts + p].n; + c->parts[p].ng = 0; + if (total == 0) { + atomic_fetch_add_explicit(&c->done, 1, memory_order_relaxed); + continue; + } + + /* Both partition-local tables are sized to 2× the record count: + * groups and distinct (k,v) pairs are each bounded by the record + * count, so load stays ≤ 0.5 and linear probing always finds an + * empty slot. */ + uint64_t dcap = 8; + while (dcap < (uint64_t)total * 2) dcap <<= 1; + uint64_t dmask = dcap - 1; + + /* (k,v) dedupe table: 16B key slot + int32 occupancy (an in-band + * sentinel is unsafe — every int64 is a valid value). */ + ray_t* dh_hdr = NULL; + int64_t* dkv = (int64_t*)scratch_alloc( + &dh_hdr, (size_t)dcap * 2 * sizeof(int64_t) + (size_t)dcap * 4); + if (!dkv) { + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + return; + } + int32_t* docc = (int32_t*)(dkv + (size_t)dcap * 2); + memset(docc, 0xFF, (size_t)dcap * 4); + + /* k table: slots of (k, group index), both written on first insert. */ + ray_t* kh_hdr = NULL; + int64_t* kk = (int64_t*)scratch_alloc( + &kh_hdr, (size_t)dcap * sizeof(int64_t) + (size_t)dcap * 4); + if (!kk) { + scratch_free(dh_hdr); + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + return; + } + int32_t* kidx = (int32_t*)(kk + dcap); + memset(kidx, 0xFF, (size_t)dcap * 4); + + int64_t cap = total < 256 ? total : 256; + cdf_grp_t* grps = (cdf_grp_t*)ray_alloc_raw((size_t)cap * sizeof(cdf_grp_t)); + if (!grps) { + scratch_free(dh_hdr); + scratch_free(kh_hdr); + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + return; + } + int64_t ng = 0; + int failed = 0; + + for (uint32_t w = 0; w < c->nw && !failed; w++) { + cdf_buf_t* b = &c->bufs[(size_t)w * c->n_parts + p]; + const char* rec = b->buf; + for (uint32_t i = 0; i < b->n; i++, rec += CDF_REC) { + uint64_t ph = ((const uint64_t*)rec)[0]; + int64_t k = ((const int64_t*)rec)[1]; + int64_t v = ((const int64_t*)rec)[2]; + int64_t row = ((const int64_t*)rec)[3]; + + /* k table probe for EVERY record: this partition's first_row + * must be the MIN over all its rows of the key, not only over + * rows that open a fresh (k,v) pair. The slot index comes + * from a hash of k ALONE — recomputed here rather than stored, + * to keep the record at 32B — and that hash never selects a + * partition, so it needs no bit shift-out. */ + uint64_t t = cdf_fmix64(ray_hash_i64(k)) & dmask; + int64_t gi = 0; + for (;;) { + if (kidx[t] == -1) { + if (ng >= cap) { + /* Invariant: ng <= total — every group is opened + * by a distinct record of this partition, so the + * record count bounds the group count and `total` + * is a legitimate clamp. The growth must never + * be a no-op while ng == cap (that would write one + * past the end), so nc is always > cap: clamp to + * total only while cap is still below it. */ + int64_t nc = cap * 2; + if (nc > total && cap < total) nc = total; + cdf_grp_t* ngp = (cdf_grp_t*)ray_realloc_raw( + grps, (size_t)nc * sizeof(cdf_grp_t)); + if (!ngp) { failed = 1; break; } + grps = ngp; + cap = nc; + } + if (ng > INT32_MAX) { failed = 1; break; } + kk[t] = k; + kidx[t] = (int32_t)ng; + grps[ng].key = k; + grps[ng].cnt = 0; + grps[ng].first = row; + gi = ng++; + break; + } + if (kk[t] == k) { + gi = kidx[t]; + if (row < grps[gi].first) grps[gi].first = row; + break; + } + t = (t + 1) & dmask; + } + if (failed) break; + + /* Dedupe on the pair hash already in the record; slot bits + * above the partition bits. */ + uint64_t s = (ph >> c->part_bits) & dmask; + for (;;) { + if (docc[s] == -1) { + dkv[s * 2] = k; + dkv[s * 2 + 1] = v; + docc[s] = 1; + grps[gi].cnt++; + break; + } + if (dkv[s * 2] == k && dkv[s * 2 + 1] == v) break; + s = (s + 1) & dmask; + } + } + } + + scratch_free(dh_hdr); + scratch_free(kh_hdr); + if (failed) { + ray_free_raw(grps); + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + return; + } + + /* Re-lay the partials out bucket by bucket (counting sort over 64 + * buckets) so phase 3 reads each bucket as one contiguous run. */ + int64_t* boff = c->parts[p].boff; + int64_t pos[CDF_MERGE_PARTS]; + memset(boff, 0, sizeof(int64_t) * (CDF_MERGE_PARTS + 1)); + for (int64_t i = 0; i < ng; i++) + boff[cdf_bucket(cdf_fmix64(ray_hash_i64(grps[i].key))) + 1]++; + for (uint32_t m = 0; m < CDF_MERGE_PARTS; m++) { + boff[m + 1] += boff[m]; + pos[m] = boff[m]; + } + cdf_grp_t* sorted = (cdf_grp_t*)ray_alloc_raw((size_t)ng * sizeof(cdf_grp_t)); + if (!sorted) { + ray_free_raw(grps); + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + return; + } + for (int64_t i = 0; i < ng; i++) + sorted[pos[cdf_bucket(cdf_fmix64(ray_hash_i64(grps[i].key)))]++] = grps[i]; + ray_free_raw(grps); + + c->parts[p].g = sorted; + c->parts[p].ng = ng; + atomic_fetch_add_explicit(&c->done, 1, memory_order_relaxed); + } +} + +/* ══════════════════════════════════════════ + * Phase 3 — parallel key-bucketed merge + * ══════════════════════════════════════════ */ + +typedef struct { + cdf_grp_t* g; + int64_t n; +} cdf_merge_t; + +typedef struct { + cdf_part_t* parts; + uint32_t n_parts; + cdf_merge_t* mg; /* [CDF_MERGE_PARTS] */ + _Atomic(int) oom; + _Atomic(int64_t) done; /* same dropped-task guard as phases 1-2 */ +} cdf_p3_ctx_t; + +static void cdf_p3_fn(void* vctx, uint32_t wid, int64_t start, int64_t end) { + (void)wid; + cdf_p3_ctx_t* c = (cdf_p3_ctx_t*)vctx; + if (atomic_load_explicit(&c->oom, memory_order_relaxed)) return; + for (int64_t m = start; m < end; m++) { + int64_t total = 0; + for (uint32_t p = 0; p < c->n_parts; p++) + total += c->parts[p].boff[m + 1] - c->parts[p].boff[m]; + c->mg[m].n = 0; + if (total == 0) { + atomic_fetch_add_explicit(&c->done, 1, memory_order_relaxed); + continue; + } + + /* Slot table sized to 2× the triple count — distinct keys in this + * bucket are bounded by it, so load stays ≤ 0.5. */ + uint64_t scap = 8; + while (scap < (uint64_t)total * 2) scap <<= 1; + uint64_t smask = scap - 1; + ray_t* s_hdr = NULL; + int32_t* slot = (int32_t*)scratch_alloc(&s_hdr, (size_t)scap * sizeof(int32_t)); + if (!slot) { + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + return; + } + memset(slot, 0xFF, (size_t)scap * sizeof(int32_t)); + + int64_t cap = total < 256 ? total : 256, n = 0; + cdf_grp_t* out = (cdf_grp_t*)ray_alloc_raw((size_t)cap * sizeof(cdf_grp_t)); + if (!out) { + scratch_free(s_hdr); + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + return; + } + int failed = 0; + + for (uint32_t p = 0; p < c->n_parts && !failed; p++) { + const cdf_grp_t* src = c->parts[p].g; + for (int64_t i = c->parts[p].boff[m]; i < c->parts[p].boff[m + 1]; i++) { + int64_t key = src[i].key; + uint64_t s = cdf_fmix64(ray_hash_i64(key)) & smask; + while (slot[s] != -1 && out[slot[s]].key != key) s = (s + 1) & smask; + if (slot[s] != -1) { + cdf_grp_t* dst = &out[slot[s]]; + /* Counts ADD: a (k,v) pair hashes to exactly one phase-1 + * partition, so partitions hold disjoint distinct sets for + * the key. first_row is a MIN, which composes likewise. */ + dst->cnt += src[i].cnt; + if (src[i].first < dst->first) dst->first = src[i].first; + continue; + } + if (n >= cap) { + int64_t nc = cap * 2; + if (nc > total && cap < total) nc = total; + cdf_grp_t* no = (cdf_grp_t*)ray_realloc_raw( + out, (size_t)nc * sizeof(cdf_grp_t)); + if (!no) { failed = 1; break; } + out = no; + cap = nc; + } + if (n > INT32_MAX) { failed = 1; break; } + out[n] = src[i]; + slot[s] = (int32_t)n; + n++; + } + } + + scratch_free(s_hdr); + if (failed) { + ray_free_raw(out); + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + return; + } + c->mg[m].g = out; + c->mg[m].n = n; + atomic_fetch_add_explicit(&c->done, 1, memory_order_relaxed); + } +} + +/* ══════════════════════════════════════════ + * Assembly + * ══════════════════════════════════════════ */ + +/* Merged group. first_row values are unique across merged groups — one entry + * per distinct key, each carrying that key's global minimum row — so the + * ordering sort is total and the comparator needs no tiebreak. */ +static int cdf_grp_cmp(const void* a, const void* b) { + int64_t x = ((const cdf_grp_t*)a)->first, y = ((const cdf_grp_t*)b)->first; + return x < y ? -1 : (x > y ? 1 : 0); +} + +/* Same sqrt-style sizing as agg_radix_part_count: at least one partition per + * worker, and enough partitions that each holds ~sqrt(nrows) records — capped + * at CDF_MAX_PARTS to bound phase 3's merge input. */ +static uint32_t cdf_part_count(uint32_t nworkers, int64_t nrows) { + uint32_t n = 1; + uint64_t rows = nrows > 0 ? (uint64_t)nrows : 1; + while ((n < nworkers || (uint64_t)n < rows / n + (rows % n != 0)) && + n < CDF_MAX_PARTS) + n <<= 1; + return n; +} + +static int cdf_type_ok(int8_t t) { + return t == RAY_I64 || t == RAY_I32 || t == RAY_I16 || RAY_IS_SYM(t); +} + +/* Free every phase-1/2 buffer; used by both the fallback and success paths. */ +static void cdf_free_all(cdf_buf_t* bufs, size_t nbuf, cdf_part_t* parts, + uint32_t n_parts) { + if (bufs) + for (size_t i = 0; i < nbuf; i++) ray_free_raw(bufs[i].buf); + ray_free_raw(bufs); + if (parts) + for (uint32_t p = 0; p < n_parts; p++) ray_free_raw(parts[p].g); + ray_free_raw(parts); +} + +ray_t* ray_cd_fused(ray_t* key_col, ray_t* val_col, int64_t nrows) { + if (!key_col || !val_col || nrows <= 0) return NULL; + if (!ray_is_vec(key_col) || !ray_is_vec(val_col)) return NULL; + if (!cdf_type_ok(key_col->type) || !cdf_type_ok(val_col->type)) return NULL; + if ((key_col->attrs & RAY_ATTR_HAS_NULLS) || + (val_col->attrs & RAY_ATTR_HAS_NULLS)) + return NULL; + if (key_col->len < nrows || val_col->len < nrows) return NULL; + if (nrows < CDF_MIN_ROWS) return NULL; /* small: existing path fine */ + + ray_pool_t* pool = ray_pool_get(); + /* Self-contained dispatch guard (folds ray_pool_par_dispatch_ok in, not + * just min-rows): live pool with background workers and not already + * inside an in-flight dispatch, so any future caller — not just the + * query.c rewrite, which currently duplicates this check — gets a safe + * kernel. The query.c call-site guard is left in place unchanged; this + * makes it redundant there but load-bearing for other callers. */ + if (!ray_pool_par_dispatch_ok(pool, nrows, CDF_MIN_ROWS)) return NULL; + + /* Memory admission gate: peak footprint is ~CDF_BYTES_PER_ROW bytes/row + * (see the constant's derivation above and the phase-3 KNOWN MEMORY + * BOUND comment). Decline rather than risk the OOM killer when that + * estimate would exceed a quarter of the heap's anon watermark (default: + * total physical RAM) — leaving headroom for the rest of the query + * (source columns, other operators, concurrent work) sharing the same + * budget. No new env knob: this rides the existing watermark, which + * ray_heap_set_anon_watermark already lets tests/embedders override. + * Same "unknown RAM -> stay permissive" convention as heap.c's own + * heap_anon_would_exceed: a caller that never ran ray_runtime_new (unit + * tests calling ray_cd_fused directly, without the app's runtime/RAM + * probe) sees ray_sys_total_ram() == 0, which must not read as a + * zero-byte budget. */ + int64_t wm = ray_heap_anon_watermark(); + if (wm > 0 && (double)nrows * CDF_BYTES_PER_ROW > (double)wm / 4.0) + return NULL; + + uint32_t nw = ray_pool_total_workers(pool); + + uint32_t n_parts = cdf_part_count(nw, nrows); + size_t nbuf = (size_t)nw * n_parts; + cdf_buf_t* bufs = (cdf_buf_t*)ray_calloc_raw(nbuf * sizeof(cdf_buf_t)); + cdf_part_t* parts = (cdf_part_t*)ray_calloc_raw((size_t)n_parts * sizeof(cdf_part_t)); + if (!bufs || !parts) { + cdf_free_all(bufs, nbuf, parts, n_parts); + return NULL; + } + + cdf_p1_ctx_t p1 = { + .kdata = ray_data(key_col), .ktype = key_col->type, .kattrs = key_col->attrs, + .vdata = ray_data(val_col), .vtype = val_col->type, .vattrs = val_col->attrs, + .n_parts = n_parts, .nw = nw, .bufs = bufs, + /* Uniform-hash expectation with 25% slack; ≥8 so tiny buffers don't + * immediately re-double. */ + .prime = (uint32_t)((uint64_t)nrows / ((uint64_t)nw * n_parts) * 5 / 4 + 8), + .oom = 0, .rows_done = 0, + }; + ray_pool_dispatch(pool, cdf_p1_fn, &p1, nrows); + if (atomic_load_explicit(&p1.oom, memory_order_relaxed) || + atomic_load_explicit(&p1.rows_done, memory_order_relaxed) != nrows) { + cdf_free_all(bufs, nbuf, parts, n_parts); + return NULL; /* fallback, not an error */ + } + + cdf_p2_ctx_t p2 = { + .bufs = bufs, .parts = parts, .nw = nw, .n_parts = n_parts, + .part_bits = (uint32_t)__builtin_ctz(n_parts), .oom = 0, .done = 0, + }; + ray_pool_dispatch_n(pool, cdf_p2_fn, &p2, n_parts); + if (atomic_load_explicit(&p2.oom, memory_order_relaxed) || + atomic_load_explicit(&p2.done, memory_order_relaxed) != (int64_t)n_parts) { + cdf_free_all(bufs, nbuf, parts, n_parts); + return NULL; + } + + /* Phase 3: merge the per-partition partials into global per-key totals, one + * task per key bucket. Keys are disjoint across buckets, so the tasks + * never touch the same entry and need no coordination. + * + * KNOWN MEMORY BOUND: peak footprint scales with ROWS, not with groups — + * phase 2's partial triples (24B per distinct (partition, key) pair, itself + * bounded by the row count) coexist with phase 1's 32B-per-row records + * until the cdf_free_all below, so a pathological all-distinct input peaks + * near 56B/row. Cardinality-aware admission (declining when the estimated + * distinct-pair count would blow a budget) is future work. */ + cdf_merge_t mg[CDF_MERGE_PARTS]; + memset(mg, 0, sizeof(mg)); + cdf_p3_ctx_t p3 = { + .parts = parts, .n_parts = n_parts, .mg = mg, .oom = 0, .done = 0, + }; + ray_pool_dispatch_n(pool, cdf_p3_fn, &p3, CDF_MERGE_PARTS); + if (atomic_load_explicit(&p3.oom, memory_order_relaxed) || + atomic_load_explicit(&p3.done, memory_order_relaxed) != CDF_MERGE_PARTS) { + for (uint32_t m = 0; m < CDF_MERGE_PARTS; m++) ray_free_raw(mg[m].g); + cdf_free_all(bufs, nbuf, parts, n_parts); + return NULL; + } + cdf_free_all(bufs, nbuf, parts, n_parts); + + int64_t ng = 0; + for (uint32_t m = 0; m < CDF_MERGE_PARTS; m++) ng += mg[m].n; + cdf_grp_t* merged = (cdf_grp_t*)ray_alloc_raw((size_t)(ng > 0 ? ng : 1) * + sizeof(cdf_grp_t)); + if (!merged) { + for (uint32_t m = 0; m < CDF_MERGE_PARTS; m++) ray_free_raw(mg[m].g); + return NULL; + } + int64_t mo = 0; + for (uint32_t m = 0; m < CDF_MERGE_PARTS; m++) { + if (mg[m].n) + memcpy(merged + mo, mg[m].g, (size_t)mg[m].n * sizeof(cdf_grp_t)); + mo += mg[m].n; + ray_free_raw(mg[m].g); + } + /* Global stable first-seen order. */ + qsort(merged, (size_t)ng, sizeof(cdf_grp_t), cdf_grp_cmp); + + ray_t* keys = ray_vec_new(RAY_I64, ng); + ray_t* cnts = ray_vec_new(RAY_I64, ng); + ray_t* firsts = ray_vec_new(RAY_I64, ng); + ray_t* tbl = ray_table_new(3); + if (!keys || RAY_IS_ERR(keys) || !cnts || RAY_IS_ERR(cnts) || + !firsts || RAY_IS_ERR(firsts) || !tbl || RAY_IS_ERR(tbl)) { + ray_release(keys); ray_release(cnts); ray_release(firsts); ray_release(tbl); + ray_free_raw(merged); + return NULL; + } + keys->len = cnts->len = firsts->len = ng; + int64_t* kd = (int64_t*)ray_data(keys); + int64_t* cd = (int64_t*)ray_data(cnts); + int64_t* fd = (int64_t*)ray_data(firsts); + for (int64_t i = 0; i < ng; i++) { + kd[i] = merged[i].key; + cd[i] = merged[i].cnt; + fd[i] = merged[i].first; + } + ray_free_raw(merged); + + tbl = ray_table_add_col(tbl, ray_sym_intern("k", 1), keys); + tbl = ray_table_add_col(tbl, ray_sym_intern("u", 1), cnts); + tbl = ray_table_add_col(tbl, ray_sym_intern("_first", 6), firsts); + ray_release(keys); + ray_release(cnts); + ray_release(firsts); + if (!tbl || RAY_IS_ERR(tbl)) { + ray_release(tbl); + return NULL; + } + return tbl; +} diff --git a/src/ops/cdfuse.h b/src/ops/cdfuse.h new file mode 100644 index 00000000..a141b71d --- /dev/null +++ b/src/ops/cdfuse.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2025-2026 Anton Kundenko + * All rights reserved. + + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#ifndef RAY_OPS_CDFUSE_H +#define RAY_OPS_CDFUSE_H + +#include "rayforce.h" + +/* Minimum row count for the fused kernel; below it the existing group + + * count-distinct rewrite is already fast enough and the scatter buffers + * are not worth their allocation. */ +#define CDF_MIN_ROWS 262144 + +/* Fused grouped count-distinct: for each distinct value of key_col, + * count the distinct values of val_col over rows [0, nrows) (all rows; + * selection handling is the caller's job — Task 4 only wires the + * unfiltered path and declines otherwise). + * Returns a RAY_TABLE with three I64 columns in this exact order: + * k[out_n], u[out_n], _first[out_n] + * (keys, distinct counts, first source row) in stable first-seen key + * order, or NULL when the shape is unsupported (caller falls back to the + * existing rewrite). key_col/val_col must be flat vectors of + * I64/I32/I16/SYM with no nulls. */ +ray_t* ray_cd_fused(ray_t* key_col, ray_t* val_col, int64_t nrows); + +#endif /* RAY_OPS_CDFUSE_H */ diff --git a/src/ops/collection.c b/src/ops/collection.c index 798096d7..c25c45f9 100644 --- a/src/ops/collection.c +++ b/src/ops/collection.c @@ -1772,6 +1772,12 @@ ray_t* ray_take_fn(ray_t* vec, ray_t* n_obj) { return ray_error("type", "take: range take unsupported for %s", ray_type_name(vec->type)); } + /* Every scalar-count branch below turns the count into a magnitude with + * `n < 0 ? -n : n`; `-INT64_MIN` is signed-overflow UB and |INT64_MIN| is + * unallocatable anyway, so reject an INT64_MIN count (also the i64 null + * sentinel) once here, up front, for all of them. */ + if (ray_is_atom(n_obj) && is_numeric(n_obj) && as_i64(n_obj) == INT64_MIN) + return ray_error("range", "take: count magnitude out of range"); /* Char take: (take 'a' n) → string of n copies of char */ if (ray_is_atom(vec) && vec->type == -RAY_STR && ray_str_len(vec) == 1 && ray_is_atom(n_obj) && is_numeric(n_obj)) { int64_t n = as_i64(n_obj); @@ -2015,7 +2021,10 @@ ray_t* ray_drop_fn(ray_t* vec, ray_t* n_obj) { start = n < len ? n : len; amount = len - start; } else { - int64_t cut = -n; + /* `-INT64_MIN` is signed-overflow UB; a drop-from-end of that + * magnitude removes the whole collection (|n| >= len), so treat it as + * cut == len (amount 0) rather than negating. */ + int64_t cut = (n == INT64_MIN) ? len : -n; amount = cut < len ? len - cut : 0; } return collection_slice(vec, start, amount); diff --git a/src/ops/exec.c b/src/ops/exec.c index cb15929e..9035d1e0 100644 --- a/src/ops/exec.c +++ b/src/ops/exec.c @@ -508,6 +508,26 @@ void partitioned_gather(ray_pool_t* pool, const int64_t* idx, int64_t n, /* Forward declarations — exec_node wraps exec_node_inner with profiling */ /* exec_node declared extern in exec_internal.h */ +/* Contiguous LIST slice with per-cell retain — the raw byte copy the HEAD/TAIL + * trims use for flat columns would alias the source's ray_t* cells without a + * refcount and double-free on release (issue #355; same reason exec_filter_head + * gathers LIST columns separately). col_vec_new/typed_vec_new also reject + * RAY_LIST, so without this the column comes back NULL and ray_table_add_col + * fails with "column must be a vector". Caller owns the returned list. */ +static ray_t* list_slice_retain(ray_t* col, int64_t start, int64_t n) { + ray_t* out = ray_list_new(n > 0 ? n : 1); + if (!out || RAY_IS_ERR(out)) return out; + out->len = n; + ray_t** d = (ray_t**)ray_data(out); + ray_t** s = (ray_t**)ray_data(col); + for (int64_t i = 0; i < n; i++) { + ray_t* e = s[start + i]; + if (e) ray_retain(e); + d[i] = e; + } + return out; +} + static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op); @@ -2931,6 +2951,10 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { } result = ray_table_add_col(result, name_id, head_vec); ray_release(head_vec); + } else if (col->type == RAY_LIST) { + ray_t* lcol = list_slice_retain(col, 0, n); + result = ray_table_add_col(result, name_id, lcol); + ray_release(lcol); } else { /* Flat column: direct copy */ uint8_t esz = col_esz(col); @@ -2944,6 +2968,10 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { * dictionary (sym-domain Phase 2) */ if (col->type == RAY_SYM) ray_sym_vec_adopt_domain(head_vec, col); + /* copied ray_str_t descriptors still reference + * the source's pool by offset (issue #404) */ + if (col->type == RAY_STR) + col_propagate_str_pool(head_vec, col); } result = ray_table_add_col(result, name_id, head_vec); ray_release(head_vec); @@ -2954,6 +2982,11 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { } if (n > input->len) n = input->len; /* Materialized copy for vector head */ + if (input->type == RAY_LIST) { + ray_t* lres = list_slice_retain(input, 0, n); + ray_release(input); + return lres; + } uint8_t esz = col_esz(input); ray_t* result = col_vec_new(input, n); if (result && !RAY_IS_ERR(result)) { @@ -2962,6 +2995,8 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { col_propagate_nulls_range(result, 0, input, 0, n); if (input->type == RAY_SYM) ray_sym_vec_adopt_domain(result, input); + if (input->type == RAY_STR) + col_propagate_str_pool(result, input); } ray_release(input); return result; @@ -3064,6 +3099,10 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { } result = ray_table_add_col(result, name_id, tail_vec); ray_release(tail_vec); + } else if (col->type == RAY_LIST) { + ray_t* lcol = list_slice_retain(col, skip, n); + result = ray_table_add_col(result, name_id, lcol); + ray_release(lcol); } else { /* Flat column: direct copy */ uint8_t esz = col_esz(col); @@ -3077,6 +3116,10 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { /* raw SYM cell-id copy — adopt source dict */ if (col->type == RAY_SYM) ray_sym_vec_adopt_domain(tail_vec, col); + /* copied ray_str_t descriptors still reference + * the source's pool by offset (issue #404) */ + if (col->type == RAY_STR) + col_propagate_str_pool(tail_vec, col); } result = ray_table_add_col(result, name_id, tail_vec); ray_release(tail_vec); @@ -3087,6 +3130,11 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { } if (n > input->len) n = input->len; int64_t skip = input->len - n; + if (input->type == RAY_LIST) { + ray_t* lres = list_slice_retain(input, skip, n); + ray_release(input); + return lres; + } uint8_t esz = col_esz(input); ray_t* result = col_vec_new(input, n); if (result && !RAY_IS_ERR(result)) { @@ -3097,6 +3145,8 @@ static ray_t* exec_node_inner(ray_graph_t* g, ray_op_t* op) { col_propagate_nulls_range(result, 0, input, skip, n); if (input->type == RAY_SYM) ray_sym_vec_adopt_domain(result, input); + if (input->type == RAY_STR) + col_propagate_str_pool(result, input); } ray_release(input); return result; diff --git a/src/ops/expr.c b/src/ops/expr.c index 08061f65..b458aa93 100644 --- a/src/ops/expr.c +++ b/src/ops/expr.c @@ -42,6 +42,49 @@ static inline uint8_t truthy_f64ish(double v) { return (v == v && v != 0.0) ? 1 : 0; } +static inline bool floor_idiv_i64_checked(int64_t a, int64_t b, int64_t* out) { + if (b == 0 || (a == INT64_MIN && b == -1)) return false; + int64_t q = a / b; + int64_t rem = a % b; + if (rem != 0 && ((a < 0) != (b < 0))) q--; + *out = q; + return true; +} + +/* Integer floor-division can be delegated to the vectorizable double kernel + * only while both operands stay within ±2^53. Such integers are exact + * doubles, and (proof) round(a/b) never crosses an integer boundary: a + * non-integer a/b is at least 1/|b| from any integer, while the rounded + * result's half-ULP is at most 1/|b| — with 2^e ≤ |a/b| we have + * |b|·2^e ≤ |a| ≤ 2^53, so 2^(e-53) ≤ 1/|b|. Thus floor of the rounded + * quotient equals the true integer floor. Outside the range the scalar + * exact kernel (floor_idiv_i64_checked) is required; NULL_I64 (INT64_MIN) + * lies far outside it, so a null in the span also falls to the exact kernel + * (which is where nulls are handled anyway). */ +#define DBL_EXACT_INT_LIM (0x1p53) /* 2^53 as a double bound */ +static inline bool i64_dbl_exact(int64_t v) { + return v <= (int64_t)DBL_EXACT_INT_LIM && v >= -(int64_t)DBL_EXACT_INT_LIM; +} +/* Gate scans are branchless accumulations rather than early-exit loops: the + * gate is paid on every morsel, so the passing (common) case — which reads the + * whole span regardless — must stay auto-vectorized. */ +static inline bool i64_span_dbl_exact(const int64_t* v, int64_t n) { + int bad = 0; + for (int64_t i = 0; i < n; i++) bad |= !i64_dbl_exact(v[i]); + return !bad; +} +/* Both operands in one pass — one traversal, two streams. */ +static inline bool i64_span2_dbl_exact(const int64_t* a, const int64_t* b, int64_t n) { + int bad = 0; + /* Two statements, not `!x | !y`: clang's -Wbitwise-instead-of-logical + * rejects `|` on two boolean operands under -Werror. Still branchless. */ + for (int64_t i = 0; i < n; i++) { + bad |= !i64_dbl_exact(a[i]); + bad |= !i64_dbl_exact(b[i]); + } + return !bad; +} + static bool atom_to_numeric(ray_t* atom, double* out_f, int64_t* out_i, bool* out_is_f64) { if (!atom || !ray_is_atom(atom)) return false; switch (atom->type) { @@ -192,12 +235,10 @@ static bool eval_const_numeric_expr(ray_graph_t* g, ray_op_t* op, case OP_SUB: r = (int64_t)((uint64_t)li - (uint64_t)ri); break; case OP_MUL: r = (int64_t)((uint64_t)li * (uint64_t)ri); break; case OP_DIV: - if (ri==0) return false; - r = li/ri; if ((li^ri)<0 && r*ri!=li) r--; + if (!floor_idiv_i64_checked(li, ri, &r)) return false; break; case OP_IDIV: - if (ri==0) return false; - r = li/ri; if ((li^ri)<0 && r*ri!=li) r--; + if (!floor_idiv_i64_checked(li, ri, &r)) return false; break; case OP_MOD: if (ri==0) return false; @@ -583,11 +624,11 @@ static bool expr_null_capable(uint8_t op, int8_t dt, int8_t t1) { /* Task 7: I64 arithmetic and unary ops. * OP_ADD=20..OP_MOD=24, OP_MIN2=33, OP_MAX2=34 in the binary block. * OP_NEG=10, OP_ABS=11 in the unary block. - * OP_IDIV=49 is NOT contiguous with OP_ADD..OP_MOD and has no I64 plain - * case in exec_elementwise_binary, so it is excluded. */ + * OP_IDIV=49 is NOT contiguous with OP_ADD..OP_MOD, but has explicit I64 + * cases in both fused and fallback kernels. */ if (dt == RAY_I64 && t1 != RAY_F64 && ((op >= OP_ADD && op <= OP_MOD) || op == OP_MIN2 || op == OP_MAX2 || - op == OP_NEG || op == OP_ABS || op == OP_SIGNUM)) + op == OP_IDIV || op == OP_NEG || op == OP_ABS || op == OP_SIGNUM)) return true; if (dt == RAY_I64 && t1 == RAY_F64 && op == OP_SIGNUM) return true; @@ -1053,6 +1094,7 @@ static void expr_exec_binary(uint8_t opcode, uint8_t null_aware, int8_t dt, void case OP_SUB: for (int64_t j = 0; j < n; j++) d[j] = ray_f64_fin(a[j] - b[j]); break; case OP_MUL: for (int64_t j = 0; j < n; j++) d[j] = ray_f64_fin(a[j] * b[j]); break; case OP_DIV: for (int64_t j = 0; j < n; j++) d[j] = b[j] != 0.0 ? ray_f64_fin(a[j] / b[j]) : NULL_F64; break; + case OP_IDIV: for (int64_t j = 0; j < n; j++) d[j] = b[j] != 0.0 ? ray_f64_fin(floor(a[j] / b[j])) : NULL_F64; break; case OP_MOD: for (int64_t j = 0; j < n; j++) { if (b[j] == 0.0) { d[j] = NULL_F64; continue; } double m = fmod(a[j], b[j]); @@ -1081,13 +1123,23 @@ static void expr_exec_binary(uint8_t opcode, uint8_t null_aware, int8_t dt, void * b==0 (non-null) → NULL_I64 mirrors fallback's zero-divisor post-pass. * b==-1 && a==INT64_MIN → overflow: direct NULL_I64 (vs fallback: loop writes 0 * then zero-divisor pass marks null — same observable result). */ - case OP_DIV: for (int64_t j=0;jtype == -RAY_F64 || lhs->type == RAY_F64 || lhs->type == -RAY_F32 || lhs->type == RAY_F32)) ? l_f64 : (double)l_i64) #define RV_READ(i) (rp_f64 ? rp_f64[i] : rp_f32 ? (double)rp_f32[i] : rp_i64 ? (double)rp_i64[i] : rp_i32 ? (double)rp_i32[i] : rp_u32 ? (double)rp_u32[i] : rp_i16 ? (double)rp_i16[i] : rp_bool ? (double)rp_bool[i] : (r_scalar && (rhs->type == -RAY_F64 || rhs->type == RAY_F64 || rhs->type == -RAY_F32 || rhs->type == RAY_F32)) ? r_f64 : (double)r_i64) +#define LV_READ_I64(i) (lp_i64 ? lp_i64[i] : lp_i32 ? (int64_t)lp_i32[i] : lp_u32 ? (int64_t)lp_u32[i] : lp_i16 ? (int64_t)lp_i16[i] : lp_bool ? (int64_t)lp_bool[i] : l_i64) +#define RV_READ_I64(i) (rp_i64 ? rp_i64[i] : rp_i32 ? (int64_t)rp_i32[i] : rp_u32 ? (int64_t)rp_u32[i] : rp_i16 ? (int64_t)rp_i16[i] : rp_bool ? (int64_t)rp_bool[i] : r_i64) /* Compute once: is lhs/rhs integer-family (not float)? Used by BOOL path. */ int l_is_int = !(lp_f64 || lp_f32 || (l_scalar && @@ -3120,6 +3182,47 @@ static void binary_range(ray_op_t* op, int8_t out_type, (rhs->type == -RAY_F64 || rhs->type == RAY_F64 || rhs->type == -RAY_F32 || rhs->type == RAY_F32))); int src_is_i64_all = l_is_int && r_is_int; + /* Integer floor-div: decide once whether the whole range fits ±2^53, so the + * exact scalar int64 kernel is only paid for genuinely large magnitudes and + * the common small-value case stays on the vectorizable double kernel. */ + bool idiv_i64_small = false; + /* Set when the I64 arm carries the gate inside its own divide loop and the + * pre-pass above was skipped entirely — see `case OP_IDIV` there. */ + bool idiv_i64_inline_gate = false; + if (src_is_i64_all && op->opcode == OP_IDIV) { + /* Gate scan plumbing — per-element cost only where it is unavoidable: + * - a broadcast scalar operand holds ONE value, so it is checked once + * here instead of re-read n times through the LV/RV_READ_I64 chain; + * - a narrow vector (I32/U32/I16/BOOL/U8) cannot leave ±2^53 by + * construction — no scan at all; + * - an I64 vector is scanned through its typed data pointer, and when + * both sides are I64 vectors the two scans share one pass; + * - for the hot I64-output shapes the scan is not a pre-pass at all: + * the I64 arm folds it into the divide loop, so the column is read + * once instead of twice (a second streaming pass over a 20 M-row + * column is DRAM-bound and was the bulk of #403's `(div col k)` + * regression). + * Every variant decides exactly the same predicate. */ + const int64_t* lscan = lp_i64; + const int64_t* rscan = rp_i64; + bool l_vec = lscan || lp_i32 || lp_u32 || lp_i16 || lp_bool; + bool r_vec = rscan || rp_i32 || rp_u32 || rp_i16 || rp_bool; + idiv_i64_small = (l_vec || i64_dbl_exact(l_i64)) && + (r_vec || i64_dbl_exact(r_i64)); + /* I64 column ÷ (I64 column | broadcast int scalar), I64 output: the + * scalar side (if any) is already decided by the check above, so the + * only per-element work left rides along in the divide loop. */ + idiv_i64_inline_gate = idiv_i64_small && lscan && (rscan || !r_vec) && + (out_type == RAY_I64 || out_type == RAY_TIMESTAMP); + /* With the inline gate the span was never validated — make sure no + * later branch can read idiv_i64_small as "proven in-range". */ + if (idiv_i64_inline_gate) idiv_i64_small = false; + if (idiv_i64_small && !idiv_i64_inline_gate) { + if (lscan && rscan) idiv_i64_small = i64_span2_dbl_exact(lscan, rscan, n); + else if (lscan) idiv_i64_small = i64_span_dbl_exact(lscan, n); + else if (rscan) idiv_i64_small = i64_span_dbl_exact(rscan, n); + } + } /* Hoist out_type outside the loop. Each branch is a tight per-element kernel. */ if (out_type == RAY_F64) { @@ -3180,8 +3283,42 @@ static void binary_range(ray_op_t* op, int8_t out_type, case OP_ADD: for(int64_t i=0;iri?li:ri;}break; @@ -3195,7 +3332,15 @@ static void binary_range(ray_op_t* op, int8_t out_type, case OP_MUL: for(int64_t i=0;i=INT32_MAX?INT32_MAX:q<=INT32_MIN?INT32_MIN+1:(int32_t)q):0;} + } else { + for(int64_t i=0;iri?li:ri;}break; @@ -3208,7 +3353,14 @@ static void binary_range(ray_op_t* op, int8_t out_type, case OP_SUB: for(int64_t i=0;i=INT16_MAX?INT16_MAX:q<=INT16_MIN?INT16_MIN+1:(int16_t)q):0;} + } else { + for(int64_t i=0;iri?li:ri;}break; @@ -3221,7 +3373,14 @@ static void binary_range(ray_op_t* op, int8_t out_type, case OP_SUB: for(int64_t i=0;i=UINT8_MAX?UINT8_MAX:(uint8_t)q):0;} + } else { + for(int64_t i=0;iri?li:ri;}break; @@ -3262,6 +3421,8 @@ static void binary_range(ray_op_t* op, int8_t out_type, } #undef LV_READ #undef RV_READ +#undef LV_READ_I64 +#undef RV_READ_I64 done: if (lsym_buf) ray_free_raw(lsym_buf); if (rsym_buf) ray_free_raw(rsym_buf); diff --git a/src/ops/group.c b/src/ops/group.c index b6fd38d2..493a260c 100644 --- a/src/ops/group.c +++ b/src/ops/group.c @@ -42,6 +42,40 @@ static inline bool group_fp_type(int8_t t) { return t == RAY_F32 || t == RAY_F64; } +/* + * group_key_f64_bits -- read an F64 GROUP BY key's bits, canonicalised. + * + * Canonicalises -0.0 -> +0.0 at the key-READ boundary, before the value is + * both stored into the key slot and hashed. ray_hash_f64 normalises -0.0, + * but group_keys_equal / ght_lanes_equal compare the raw 8 bytes: a column + * carrying both bit patterns hashed +0.0 and -0.0 into the same slot chain + * yet never compared equal, so -0.0 formed its own group (#407). Normalising + * once here keeps hash and compare in lockstep on every path (the ab5053a9 + * lesson) and makes +0.0 the emitted representative of the merged group + * regardless of which row arrived first — the stored bits are +0.0 for every + * contributing row, so the result is order- and thread-independent. + * + * NaN is deliberately NOT canonicalised. Distinct NaN payloads hash + * distinctly AND compare distinctly, which is already self-consistent — the + * -0.0 defect was an asymmetry between hash and compare, and NaN has none. + * Canonicalising would also fold 0Nf (NULL_F64) together with a + * runtime-produced NaN; SQL nullness is carried by the key null mask, not by + * the payload. Bit-pattern grouping of NaNs is only observable on a column + * WITHOUT the HAS_NULLS attr: with it set, sentinel detection is `x != x`, + * so EVERY NaN payload is a null and the mask folds them into the one null + * group before the payload is ever compared. + */ +static inline int64_t group_key_f64_bits(const void* data, int64_t row) { + int64_t bits; + memcpy(&bits, (const double*)data + row, 8); + /* Bit-level, never `if (v == 0.0) v = 0.0`: release builds use + * -fno-signed-zeros, under which that form folds to a no-op (same trap + * clear_neg_zero documents). Lowers to one compare + cmov on the value + * that was loaded anyway. */ + if (bits == INT64_MIN) bits = 0; /* -0.0 -> +0.0 */ + return bits; +} + static inline double group_fp_at(const void* data, int8_t t, int64_t row) { return t == RAY_F64 ? ((const double*)data)[row] : (double)((const float*)data)[row]; @@ -2314,19 +2348,18 @@ static inline uint64_t mode_scalar_key(ray_t* src, int64_t row) { const void* base = ray_data(src); switch (src->type) { case RAY_F64: { - double v; - memcpy(&v, (const char*)base + (size_t)row * 8, 8); - if (v == 0.0) v = 0.0; + /* Bit-level -0.0 fold: `if (v == 0.0) v = 0.0` is a no-op + * under release's -fno-signed-zeros (the #407 trap), which + * made (mode f) build-dependent for +/-0.0. */ uint64_t k; - memcpy(&k, &v, 8); + memcpy(&k, (const char*)base + (size_t)row * 8, 8); + if (k == 0x8000000000000000ULL) k = 0; return k; } case RAY_F32: { - float v; - memcpy(&v, (const char*)base + (size_t)row * 4, 4); - if (v == 0.0f) v = 0.0f; uint32_t k32; - memcpy(&k32, &v, 4); + memcpy(&k32, (const char*)base + (size_t)row * 4, 4); + if (k32 == 0x80000000U) k32 = 0; return (uint64_t)k32; } case RAY_SYM: @@ -3417,11 +3450,14 @@ void ght_layout_copy(ght_layout_t* dst, const ght_layout_t* src) { } } +static inline bool ray_key_may_be_null(const ray_t* kv); + bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, ray_t** agg_vecs, ray_t** agg_vecs2, uint8_t need_flags, const uint16_t* agg_ops, - const int8_t* key_types) { + const int8_t* key_types, + ray_t* const* key_vecs) { memset(out, 0, sizeof(*out)); out->n_keys = (uint16_t)n_keys; out->n_aggs = (uint16_t)n_aggs; @@ -3455,6 +3491,33 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, out->any_inline_str = 1; } } + /* Packed-tuple eligibility: every key is a plain fixed-width integer + * lane. This is a WHITELIST of exactly the types read_col_i64 + * decodes deliberately, not a `!= RAY_F64` blacklist: read_col_i64's + * `default` arm reads ONE BYTE, so an un-enumerated fixed-width type + * (RAY_F32 is the live example) would be silently mis-loaded, and a + * packed layout would then hash those wrong bits as the key identity. + * F64 is excluded by the same rule (read_col_i64 does not decode it), + * which additionally keeps the packed lanes clear of the -0.0 + * canonicalisation the F64 key builders apply (group_key_f64_bits, + * #407) — see ght_layout_t.packed_key. + * GUID/STR are excluded by any_wide_key. n_keys == 0 is not a + * tuple. */ + if (n_keys >= 1 && !out->any_wide_key) { + bool packed = true; + for (uint32_t k = 0; k < n_keys && packed; k++) { + switch (key_types[k]) { + case RAY_BOOL: case RAY_U8: case RAY_I16: case RAY_I32: + case RAY_I64: case RAY_DATE: case RAY_TIME: + case RAY_TIMESTAMP: case RAY_SYM: + break; + default: + packed = false; + break; + } + } + out->packed_key = packed ? 1 : 0; + } } uint16_t nv = 0; @@ -3476,11 +3539,22 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, agg_ops[a] == OP_MODE || agg_ops[a] == OP_TOP_N || agg_ops[a] == OP_BOT_N || wide_mm); + /* OP_COUNT is GROUP SIZE: its emit reads the group row count (cnt), + * never a staged value or an off_nn slot (see the OP_COUNT arms of + * radix_phase3_fn / the sequential emit, and the DA path's "value is + * ignored for COUNT" skip). Reserving a value slot for it therefore + * costs 8 bytes on every fat entry and every HT group row, one extra + * source-column read per row in phase 1, and one extra 8-byte store — + * all of it dead. Give it no slot: ClickBench q16/q18 are count-only, + * so this takes their entry from 48 to 40 bytes across 10M rows. */ + bool valueless = agg_ops && agg_ops[a] == OP_COUNT; uint8_t af = 0; if (holistic) { af |= GHT_AF_HOLISTIC; if (wide_mm) af |= GHT_AF_WIDE; out->agg_val_slot[a] = -1; + } else if (valueless) { + out->agg_val_slot[a] = -1; } else if (agg_vecs[a]) { out->agg_val_slot[a] = (int8_t)nv; bool pair = agg_ops && agg_is_binary_agg(agg_ops[a]); @@ -3557,8 +3631,29 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, out->agg_flags_any = agg_any; /* Null tracking: ceil(n_keys/64) int64 words, floored at 1 so the rare * n_keys==0 HT fallback keeps a (trivially-zero) null slot — byte-identical - * to the legacy single-int64 layout for every ≤64-key shape. */ + * to the legacy single-int64 layout for every ≤64-key shape. + * + * Null-mask ELISION: the words exist solely to keep a NULL key distinct + * from a 0 / "" key. When no key column can yield a NULL there is nothing + * to distinguish and the words are pure overhead — 8 bytes on every fat + * entry AND every HT group row, copied by phase 1, re-read by phase 2, and + * folded into every key compare. On ClickBench q18 (10M rows, 4.9M + * groups, 3 null-free keys) that is 8 of 56 entry bytes. Elide when the + * caller hands us the key vectors and every one of them is provably + * null-free. Capped at 64 keys so the surviving null-mask READERS can + * substitute a single shared zero word (see ght_null_words_at). + * + * INVARIANT: every writer of the mask (`nullw[k>>6] |= …` in the phase-1 + * key builders) is guarded by the caller's own `nullable` summary, which + * is computed from the SAME key_vecs through the SAME ray_key_may_be_null + * predicate — so a layout with null_words == 0 is never written to. */ uint32_t null_words = n_keys ? ((n_keys + 63u) >> 6) : 1u; + if (key_vecs && n_keys >= 1 && n_keys <= 64) { + bool any_nullable = false; + for (uint32_t k = 0; k < n_keys && !any_nullable; k++) + any_nullable = ray_key_may_be_null(key_vecs[k]); + if (!any_nullable) null_words = 0; + } out->null_words = (uint16_t)null_words; /* Key region = keys + null_words*8 null-mask words (stored after last key). * The null-mask words hold a bitmap of which keys were null in the source @@ -3674,6 +3769,7 @@ static bool group_ht_init_sized(group_ht_t* ht, uint32_t cap, const ght_layout_t* ly, uint32_t init_grp_cap) { ht->ht_cap = cap; ht->oom = 0; + ht->grow_cap = 0; /* callers that know a bound set it right after init */ /* By-value embed: re-point the layout's bases at this HT's own inline * arrays (inline src) or borrow the shared spill (wide src). The source * layout (owned by the caller's exec_group/pivot master) outlives this HT. */ @@ -3752,6 +3848,11 @@ void group_ht_free(group_ht_t* ht) { static bool group_ht_grow(group_ht_t* ht) { uint32_t old_cap = ht->grp_cap; uint32_t new_cap = old_cap * 2; + /* Deliberately NOT accelerated by grow_cap (see group_ht_t.grow_cap): the + * row array is the memory-dominant one (row_stride is 24-48 B against a + * slot's 4 B), and growing it early buys nothing — a row-array growth is + * one realloc/memmove, whereas a SLOT growth re-hashes and re-inserts + * every live group. Plain doubling keeps this array O(groups). */ uint16_t rs = ht->layout.row_stride; char* new_rows = (char*)scratch_realloc( &ht->_h_rows, (size_t)old_cap * rs, (size_t)new_cap * rs); @@ -3801,6 +3902,157 @@ static inline uint64_t ght_hash_null_words(uint64_t h, const int64_t* nullw, return h; } +/* Avalanche a packed key tuple — `lanes` 8-byte words covering the whole key + * region (n_keys value lanes + null_words mask lanes) — into one 64-bit hash. + * + * Only for ly->packed_key layouts: every lane's stored BITS are the key's + * identity there, which is exactly the precondition for hashing the tuple as + * one value instead of hashing each key and combining. Same mixer as + * ray_hash_i64 / ray_hash_combine (wyhash's wymum + wymix), but folded over + * the tuple in a single chain: ceil(lanes/2) + 1 mixer rounds instead of + * `lanes` hashes plus `lanes-1` combines. ClickBench q18's 3-key tuple goes + * from 5 rounds (10 multiplies) to 3 (3 multiplies). + * + * Bit discipline: the final wymix is a full 128->64 avalanche, so the + * partition bits (hash >> 16, group_radix_part) and the table-slot bits + * (hash & mask) remain independent — the same guarantee ray_hash_i64 gave. + * The lane count is seeded in so tuples of different arity cannot collide by + * zero-padding. */ +static inline uint64_t ght_hash_lanes(const int64_t* lane, uint32_t lanes) { + uint64_t a = 0x2d358dccaa6c78a5ULL; + uint64_t b = 0x8bb84b93962eacc9ULL ^ (uint64_t)lanes; + uint32_t k = 0; + for (; k + 2 <= lanes; k += 2) { + a ^= (uint64_t)lane[k]; + b ^= (uint64_t)lane[k + 1]; + ray__wymum(&a, &b); + } + if (k < lanes) { + /* Odd tail lane feeds BOTH mixer halves, so wymum stays quadratic in + * the lane value. Folding it into `a` alone leaves the other half a + * constant, which degrades to a multiply-by-constant — measurably + * worse probe chains on single-key group-bys (ClickBench q15). */ + uint64_t v = (uint64_t)lane[k]; + a ^= v; + b ^= v; + ray__wymum(&a, &b); + } + return ray__wymix(a ^ 0x2d358dccaa6c78a5ULL, b ^ 0x8bb84b93962eacc9ULL); +} + +/* Gather one key column's lanes for a batch of rows. Semantically + * `dst[j] = read_col_i64(data, rows[j], type, attrs)` for every j — the type + * (and, for SYM, the stored-id-width) dispatch is hoisted OUT of the row loop, + * which is the whole point: read_col_i64's switch lowers to a jump table, so + * the row-major builder pays one indirect jump per key per row and mispredicts + * whenever a tuple mixes widths (ClickBench q18 is I64, I64, SYM32). + * + * MUST stay bit-identical to read_col_i64 (ops/internal.h) — narrow signed + * types sign-extend, SYM ids zero-extend. */ +static inline void group_keys_gather(int64_t* dst, const void* data, + const int64_t* rows, uint32_t m, + int8_t type, uint8_t attrs) { + switch (type) { + case RAY_I64: case RAY_TIMESTAMP: + for (uint32_t j = 0; j < m; j++) dst[j] = ((const int64_t*)data)[rows[j]]; + return; + case RAY_SYM: + switch (attrs & RAY_SYM_W_MASK) { + case RAY_SYM_W8: + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)((const uint8_t*)data)[rows[j]]; + return; + case RAY_SYM_W16: + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)((const uint16_t*)data)[rows[j]]; + return; + case RAY_SYM_W32: + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)((const uint32_t*)data)[rows[j]]; + return; + default: + for (uint32_t j = 0; j < m; j++) dst[j] = ((const int64_t*)data)[rows[j]]; + return; + } + case RAY_I32: case RAY_DATE: case RAY_TIME: + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)((const int32_t*)data)[rows[j]]; + return; + case RAY_I16: + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)((const int16_t*)data)[rows[j]]; + return; + default: /* RAY_BOOL, RAY_U8 */ + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)((const uint8_t*)data)[rows[j]]; + return; + } +} + +/* Contiguous-row variant of group_keys_gather: rows row0..row0+m. The + * indexed form above hides the sequential access behind rows[j], which blocks + * vectorisation; with no WHERE and no match list (the common full-scan case) + * this reads each column as a straight run and the widening loads vectorise. */ +static inline void group_keys_gather_seq(int64_t* dst, const void* data, + int64_t row0, uint32_t m, + int8_t type, uint8_t attrs) { + switch (type) { + case RAY_I64: case RAY_TIMESTAMP: { + const int64_t* p = (const int64_t*)data + row0; + for (uint32_t j = 0; j < m; j++) dst[j] = p[j]; + return; + } + case RAY_SYM: + switch (attrs & RAY_SYM_W_MASK) { + case RAY_SYM_W8: { + const uint8_t* p = (const uint8_t*)data + row0; + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)p[j]; + return; + } + case RAY_SYM_W16: { + const uint16_t* p = (const uint16_t*)data + row0; + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)p[j]; + return; + } + case RAY_SYM_W32: { + const uint32_t* p = (const uint32_t*)data + row0; + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)p[j]; + return; + } + default: { + const int64_t* p = (const int64_t*)data + row0; + for (uint32_t j = 0; j < m; j++) dst[j] = p[j]; + return; + } + } + case RAY_I32: case RAY_DATE: case RAY_TIME: { + const int32_t* p = (const int32_t*)data + row0; + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)p[j]; + return; + } + case RAY_I16: { + const int16_t* p = (const int16_t*)data + row0; + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)p[j]; + return; + } + default: { /* RAY_BOOL, RAY_U8 */ + const uint8_t* p = (const uint8_t*)data + row0; + for (uint32_t j = 0; j < m; j++) dst[j] = (int64_t)p[j]; + return; + } + } +} + +/* Shared all-zero stand-in for an elided null-mask region (null_words == 0, + * i.e. no key column can produce a NULL). Elision is capped at 64 keys, so a + * single word covers every (k >> 6) a reader can form. */ +static const int64_t ght_null_words_none[1] = { 0 }; + +/* Base of a group row's / entry's null-mask words. `keybase` is the start of + * the key region (row + 8 or entry + 8). Returns the shared zero word when + * the layout elided the mask, so readers stay branch-free and never step past + * the key region into the accumulator block. */ +static inline const int64_t* ght_null_words_at(const ght_layout_t* ly, + const void* keybase) { + if (!ly->null_words) return ght_null_words_none; + return (const int64_t*)(const void*)((const char*)keybase + + ly->key_off[ly->n_keys]); +} + /* True when key column kv (or its slice parent) carries the HAS_NULLS attr and * so may yield a null at some row. Replaces the old per-key `1u << k` nullable * bitmask (which silently dropped keys past index 7 / was UB past 31): callers @@ -3883,8 +4135,11 @@ static inline uint64_t inline_build_keys(const ght_layout_t* ly, const int8_t* k *(int64_t*)slot = row; kh = wide_key_hash_at(ly, k, key_data, key_pool, row); } else if (key_types[k] == RAY_F64) { - *(int64_t*)slot = ((const int64_t*)key_data[k])[row]; - kh = ray_hash_f64(((const double*)key_data[k])[row]); + /* -0.0 -> +0.0 before BOTH the store and the hash (group_key_f64_bits). */ + int64_t kv = group_key_f64_bits(key_data[k], row); + double dv; memcpy(&dv, &kv, 8); + *(int64_t*)slot = kv; + kh = ray_hash_f64(dv); } else { int64_t kv = read_col_i64(key_data[k], row, key_types[k], key_attrs[k]); *(int64_t*)slot = kv; @@ -3912,6 +4167,11 @@ static inline uint64_t hash_keys_inline(const int64_t* keys, const int8_t* key_t const int64_t* nullw = (const int64_t*)((const char*)keys + ly->key_off[n_keys]); return ght_hash_null_words(h, nullw, ly->null_words); } + /* Packed tuple: one avalanche over the whole key region. MUST stay in + * lockstep with every phase-1 key builder below — a rehash that hashed a + * row differently from the entry that created it would lose the group. */ + if (ly->packed_key) + return ght_hash_lanes(keys, (uint32_t)n_keys + ly->null_words); const uint8_t* const kflags = ly->key_flags; uint64_t h = 0; for (uint32_t k = 0; k < n_keys; k++) { @@ -3936,6 +4196,17 @@ static inline uint64_t hash_keys_inline(const int64_t* keys, const int8_t* key_t static void group_ht_rehash(group_ht_t* ht, const int8_t* key_types) { uint32_t new_cap = ht->ht_cap * 2; + /* Skip ahead toward the caller's bound (see group_ht_t.grow_cap): each + * intermediate rung re-hashes and re-inserts every live group. Capped at + * ONE extra doubling per rehash, because the only thing known here is that + * the table just crossed ht_cap/2 groups — not that it will keep growing. + * A table that stops right after the jump over-allocates its slot array by + * 2x and nothing else; without the cap it would take the full row-derived + * bound (up to 8x at 10M) on the strength of a single rung. */ + uint32_t tgt = ht->grow_cap; + uint32_t lim = ht->ht_cap << 2; + if (tgt > lim) tgt = lim; + if (tgt > new_cap) new_cap = tgt; ray_t* new_h = NULL; uint32_t* new_slots = (uint32_t*)scratch_alloc(&new_h, (size_t)new_cap * sizeof(uint32_t)); if (!new_slots) return; /* OOM: keep old HT, it still works (just slower) */ @@ -4354,19 +4625,87 @@ static void init_accum_from_entry_nullable(char* row, const char* entry, accum_from_entry_nullable(row, entry, ly); } +/* ── Lane-wise key-region copy / compare ─────────────────────────────────── + * Every group key region is a whole number of 8-byte lanes (8 B per scalar + * key, 16 B per inline-STR descriptor, plus null_words*8 trailing mask words), + * but its length is a runtime uint16 — so memcpy/memcmp over it lower to libc + * calls that re-dispatch on the size for every single row. A dwarf + * call-graph profile of ClickBench q18 (3 narrow keys, 10M rows, 4.9M groups) + * charged 11.2% of the query to __memmove_avx under radix_buf_push, a further + * 2.5% to the same under radix_phase2_fn's group-row copy, and 2.2% to + * __memcmp_avx2_movbe under group_keys_equal — every one of them moving 24 or + * 32 bytes. Narrow layouts (≤ GHT_LANES_INLINE lanes, i.e. every group-by up + * to 7 scalar keys) run a switch of straight u64 loads/stores instead: no + * call, no size dispatch, and the compare short-circuits on first mismatch. + * Wider layouts keep the libc call, so their generated code is unchanged. + * + * Alignment: key regions start at entry+8 / row+8 inside malloc'd blocks whose + * strides are all multiples of 8, so every lane is naturally aligned. */ +#define GHT_LANES_INLINE 8 + +/* Whole-lane precondition. Both helpers round the length DOWN to lanes, so a + * length that is not a multiple of 8 would silently drop the tail bytes from + * the copy / the compare. Every caller passes ly->key_region or + * n_agg_vals*8, both built purely from 8- and 16-byte pieces, so this is an + * invariant, not input validation — checked under DEBUG/RAY_HARDENED (which + * is what `make test` builds) and free in release, matching RAY_ASSERT_VALUE's + * convention. */ +#if defined(DEBUG) || defined(RAY_HARDENED) +#define GHT_LANES_WHOLE(bytes) assert((((unsigned)(bytes)) & 7u) == 0 && \ + "ght_lanes_* length must be a whole number of 8-byte lanes") +#else +#define GHT_LANES_WHOLE(bytes) ((void)0) +#endif + +static inline void ght_lanes_copy(void* dst, const void* src, uint16_t bytes) { + uint64_t* d = (uint64_t*)dst; + const uint64_t* s = (const uint64_t*)src; + GHT_LANES_WHOLE(bytes); + switch ((unsigned)bytes >> 3) { + default: memcpy(dst, src, bytes); return; + case 8: d[7] = s[7]; __attribute__((fallthrough)); + case 7: d[6] = s[6]; __attribute__((fallthrough)); + case 6: d[5] = s[5]; __attribute__((fallthrough)); + case 5: d[4] = s[4]; __attribute__((fallthrough)); + case 4: d[3] = s[3]; __attribute__((fallthrough)); + case 3: d[2] = s[2]; __attribute__((fallthrough)); + case 2: d[1] = s[1]; __attribute__((fallthrough)); + case 1: d[0] = s[0]; __attribute__((fallthrough)); + case 0: return; + } +} + +static inline bool ght_lanes_equal(const void* a, const void* b, uint16_t bytes) { + const uint64_t* x = (const uint64_t*)a; + const uint64_t* y = (const uint64_t*)b; + GHT_LANES_WHOLE(bytes); + switch ((unsigned)bytes >> 3) { + default: return memcmp(a, b, bytes) == 0; + case 8: if (x[7] != y[7]) return false; __attribute__((fallthrough)); + case 7: if (x[6] != y[6]) return false; __attribute__((fallthrough)); + case 6: if (x[5] != y[5]) return false; __attribute__((fallthrough)); + case 5: if (x[4] != y[4]) return false; __attribute__((fallthrough)); + case 4: if (x[3] != y[3]) return false; __attribute__((fallthrough)); + case 3: if (x[2] != y[2]) return false; __attribute__((fallthrough)); + case 2: if (x[1] != y[1]) return false; __attribute__((fallthrough)); + case 1: if (x[0] != y[0]) return false; __attribute__((fallthrough)); + case 0: return true; + } +} + /* Compare the n_keys key slots of two rows, handling wide keys via * key_data[] resolution. Returns true if all keys are bytewise equal. - * Hot path: when wide_mask == 0, reduces to a single memcmp over the + * Hot path: when wide_mask == 0, reduces to a lane-wise compare over the * packed 8-byte-per-key region. */ static inline bool group_keys_equal(const int64_t* a_keys, const int64_t* b_keys, const ght_layout_t* ly, void* const* key_data, const void* const* key_pool) { uint16_t nk = ly->n_keys; if (!ly->any_wide_key) { - /* memcmp covers nk 8-byte values + the null_words trailing words: + /* Covers nk 8-byte values + the null_words trailing words: * key_region == (nk + null_words)*8 with no wide/inline-STR keys, so * the compare length widens with the null region — no shape change. */ - return memcmp(a_keys, b_keys, (size_t)ly->key_region) == 0; + return ght_lanes_equal(a_keys, b_keys, ly->key_region); } const uint8_t* const kflags = ly->key_flags; const uint16_t* const koff = ly->key_off; @@ -4437,7 +4776,7 @@ static inline uint32_t group_probe_entry(group_ht_t* ht, uint32_t gid = ht->grp_count++; char* row = ht->rows + (size_t)gid * ly->row_stride; *(int64_t*)row = 1; /* count = 1 */ - memcpy(row + 8, ekeys, key_bytes); + ght_lanes_copy(row + 8, ekeys, key_bytes); if (!accum_skip) init_accum_from_entry(row, entry, ly); else if (ly->row_stride > 8 + key_bytes) @@ -4563,6 +4902,23 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, int64_t* nullw = ek + nk; uint32_t null_words = ly->null_words; for (uint32_t w = 0; w < null_words; w++) nullw[w] = 0; + if (ly->packed_key) { + /* Packed tuple (ly->packed_key): every key is a plain integer lane, so + * the wide/F64 arms of the generic loop below are dead and the whole + * key region — values plus null-mask words — avalanches in ONE + * ght_hash_lanes call instead of nk hashes plus nk-1 combines. MUST + * stay in lockstep with hash_keys_inline (rehash/merge/lookup). */ + for (uint32_t k = 0; k < nk; k++) { + if (__builtin_expect(any_nullable && ray_key_may_be_null(key_vecs[k]) + && ray_vec_is_null(key_vecs[k], row), 0)) { + nullw[k >> 6] |= (int64_t)((uint64_t)1 << (k & 63)); + ek[k] = 0; + } else { + ek[k] = read_col_i64(key_data[k], row, key_types[k], key_attrs[k]); + } + } + h = ght_hash_lanes(ek, (uint32_t)nk + null_words); + } else { for (uint32_t k = 0; k < nk; k++) { int8_t t = key_types[k]; uint64_t kh; @@ -4578,10 +4934,11 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, ek[k] = row; kh = wide_key_hash_at(ly, k, key_data, ht->key_pool, row); } else if (t == RAY_F64) { - int64_t kv; - memcpy(&kv, &((double*)key_data[k])[row], 8); + /* -0.0 -> +0.0 before BOTH the store and the hash. */ + int64_t kv = group_key_f64_bits(key_data[k], row); + double dv; memcpy(&dv, &kv, 8); ek[k] = kv; - kh = ray_hash_f64(((double*)key_data[k])[row]); + kh = ray_hash_f64(dv); } else { int64_t kv = read_col_i64(key_data[k], row, t, key_attrs[k]); ek[k] = kv; @@ -4591,15 +4948,19 @@ void group_rows_range(group_ht_t* ht, void** key_data, int8_t* key_types, } h = ght_hash_null_words(h, nullw, null_words); } + } *(uint64_t*)ebuf = h; int64_t* ev = (int64_t*)(ebuf + 8 + (size_t)ly->key_region); - uint8_t vi = 0; for (uint32_t a = 0; a < na; a++) { uint8_t af = aflags[a]; - /* Holistic agg (OP_MEDIAN): no slot reserved — skip packing. - * Source column read in the post-radix pass. */ - if (af & GHT_AF_HOLISTIC) continue; + /* Destination slot comes from the layout, never a running counter: + * aggs that reserve no value slot (holistic OP_MEDIAN &c., and + * valueless OP_COUNT) carry agg_val_slot == -1, and a running + * counter would silently shift every later agg's slot. */ + int8_t vs = ly->agg_val_slot[a]; + if (vs < 0) continue; + uint8_t vi = (uint8_t)vs; ray_t* ac = agg_vecs[a]; if (!ac) continue; if (agg_strlen && agg_strlen[a]) @@ -4670,6 +5031,196 @@ typedef struct { uint32_t gid; } group_topn_item_t; +/* ---- top-N group selection (`by … desc: take: N`) ------------------- + * The emit filter keeps only the globally best N groups before the phase-3 + * emit. Selection is a bounded heap over EVERY group of EVERY partition, so + * its cost is O(total groups) — tens of millions of rows on a high-cardinality + * 100M-row group-by, and it ran on one thread. + * + * `TOPN_BETTER(desc, a, b)` is the single ordering predicate: for `desc` a + * larger value is better, otherwise a smaller one. The heap keeps the WORST + * retained item at the root (min-heap when desc), so the invariant is + * "parent is not better than child" and a swap is needed exactly when the + * parent IS better. */ +#define TOPN_BETTER(desc_dir, a, b) ((desc_dir) ? ((a) > (b)) : ((a) < (b))) + +/* Push one candidate into a bounded "best k" heap; returns the new count. + * Replacement is on STRICT improvement: an item that merely TIES the root is + * rejected and the heap is left byte-for-byte unchanged. + * + * That is NOT the same as "the earliest of equal values wins" — with k=2, + * desc, pushing 5,5,7 evicts the FIRST 5, because sift-down may promote + * either equal child to the root. What holds is only that the outcome is a + * deterministic function of the push sequence, which is all the callers need. + * + * WHY THE PARALLEL PRE-PASS RETAINS THE SAME SET AS ONE SERIAL SCAN. Let H + * be the global heap of the serial scan and H_p a partition's local heap. + * When H_p is full it holds the k best of what that partition has shown so + * far; H at the same point holds the k best of a SUPERSET of those items, so + * H's root is never worse than H_p's. An item H_p rejects is therefore not + * strictly better than H's root either — H would reject it too, leaving H + * unchanged. So the pre-pass elides only pushes that are provably no-ops, + * and feeding H the survivors alone reproduces its exact trajectory. + * + * TWO CONDITIONS CARRY THAT ARGUMENT. Do not "optimise" either away: + * (1) the per-partition cap is k_take ITSELF, never k_take/n_parts or any + * other share. (It may be smaller only when the partition holds fewer + * than k_take groups — then the heap drops nothing at all.) With a + * smaller cap H_p's root could be BETTER than H's, and H_p would drop + * an item the serial scan keeps. + * (2) the merge feeds candidates in the serial scan's order: partitions + * ascending, and gid ascending within a partition. Ties are + * order-sensitive (see the 5,5,7 trace above), so topn_scan_fn sorts + * its survivors back into gid order for exactly this reason. */ +static inline int64_t topn_heap_push(group_topn_item_t* h, int64_t hn, + int64_t k, uint8_t desc_dir, int64_t v, + uint32_t part, uint32_t gid) { + if (hn < k) { + int64_t j = hn++; + h[j] = (group_topn_item_t){v, part, gid}; + while (j > 0) { /* sift up */ + int64_t pr = (j - 1) >> 1; + if (!TOPN_BETTER(desc_dir, h[pr].value, h[j].value)) break; + group_topn_item_t t = h[pr]; h[pr] = h[j]; h[j] = t; + j = pr; + } + return hn; + } + if (!TOPN_BETTER(desc_dir, v, h[0].value)) return hn; + h[0] = (group_topn_item_t){v, part, gid}; + for (int64_t j = 0;;) { /* sift down */ + int64_t l = j * 2 + 1, r = l + 1, m = j; + if (l < hn && TOPN_BETTER(desc_dir, h[m].value, h[l].value)) m = l; + if (r < hn && TOPN_BETTER(desc_dir, h[m].value, h[r].value)) m = r; + if (m == j) break; + group_topn_item_t t = h[m]; h[m] = h[j]; h[j] = t; + j = m; + } + return hn; +} + +static int topn_gid_cmp(const void* a, const void* b) { + uint32_t ga = ((const group_topn_item_t*)a)->gid; + uint32_t gb = ((const group_topn_item_t*)b)->gid; + return ga < gb ? -1 : (ga > gb ? 1 : 0); +} + +static int topn_part_gid_cmp(const void* a, const void* b) { + const group_topn_item_t* x = (const group_topn_item_t*)a; + const group_topn_item_t* y = (const group_topn_item_t*)b; + if (x->part != y->part) return x->part < y->part ? -1 : 1; + return x->gid < y->gid ? -1 : (x->gid > y->gid ? 1 : 0); +} + +/* Parallel pre-pass: reduce each partition to its OWN top-k. The global + * top-k is a subset of the union of those, so the serial merge that follows + * only walks the survivors. Each worker sorts its survivors back into gid + * order and the merge walks partitions in ascending order, so the merge sees + * candidates in the same (partition, gid) sequence the single-threaded scan + * did — the retained set, and therefore the emitted rows, are identical. */ +typedef struct { + group_ht_t* part_hts; + group_topn_item_t* items; /* [item_off[n_parts]] staging */ + const uint32_t* item_off; /* [n_parts + 1] per-partition capacity */ + uint32_t* item_cnt; /* [n_parts] survivors produced */ + uint16_t order_off; /* in-row byte offset of the ordering agg */ + uint8_t desc_dir; +} topn_scan_ctx_t; + +static void topn_scan_fn(void* vctx, uint32_t worker_id, int64_t start, + int64_t end) { + (void)worker_id; + topn_scan_ctx_t* c = (topn_scan_ctx_t*)vctx; + for (int64_t pi = start; pi < end; pi++) { + uint32_t p = (uint32_t)pi; + group_ht_t* ph = &c->part_hts[p]; + uint32_t gc = ph->grp_count; + uint16_t rs = ph->layout.row_stride; + group_topn_item_t* h = c->items + c->item_off[p]; + int64_t cap = (int64_t)(c->item_off[p + 1] - c->item_off[p]); + int64_t hn = 0; + for (uint32_t gi = 0; gi < gc; gi++) { + int64_t v = *(const int64_t*)(const void*) + (ph->rows + (size_t)gi * rs + c->order_off); + hn = topn_heap_push(h, hn, cap, c->desc_dir, v, p, gi); + } + if (hn > 1) qsort(h, (size_t)hn, sizeof(*h), topn_gid_cmp); + c->item_cnt[p] = (uint32_t)hn; + } +} + +/* FUSED per-partition scan: the same reduction topn_scan_fn performs, run at + * the END of the phase-2 task that BUILT the partition, while its rows are + * still in that core's L2/L3 instead of being re-read from DRAM afterwards. + * + * It is arithmetically the same scan — same shared topn_heap_push, the same + * per-partition cap (`cap` is k_take ITSELF, condition (1) of the proof above, + * here as a uniform stride because grp_count is unknown at allocation time), + * and the same sort back into gid order so the merge still sees candidates + * partitions-ascending / gid-ascending (condition (2)). Fusion moves only + * WHERE the scan runs, never what it retains. + * + * `items == NULL` means fusion is off and the standalone topn_scan_fn pre-pass + * (or the serial scan) runs instead. `item_cnt` is calloc'd by the driver and + * every phase-2 task clears its own partition's entry BEFORE any early + * `continue`, so a partition that is skipped — or whose task never runs at all + * because the dispatch was drained by a cancel — reads as 0 survivors. */ +typedef struct { + group_topn_item_t* items; /* [n_parts * cap]; NULL = fusion off */ + uint32_t* item_cnt; /* [n_parts] survivors per partition */ + uint32_t cap; /* per-partition capacity = k_take */ + uint16_t order_off; /* in-row byte offset of the ordering agg */ + uint8_t desc_dir; +} topn_fuse_t; + +static inline void topn_fuse_partition(const topn_fuse_t* f, group_ht_t* ph, + uint32_t p) { + uint32_t gc = ph->grp_count; + uint16_t rs = ph->layout.row_stride; + group_topn_item_t* h = f->items + (size_t)p * f->cap; + int64_t cap = (int64_t)f->cap; + int64_t hn = 0; + for (uint32_t gi = 0; gi < gc; gi++) { + int64_t v = *(const int64_t*)(const void*) + (ph->rows + (size_t)gi * rs + f->order_off); + hn = topn_heap_push(h, hn, cap, f->desc_dir, v, p, gi); + } + if (hn > 1) qsort(h, (size_t)hn, sizeof(*h), topn_gid_cmp); + f->item_cnt[p] = (uint32_t)hn; +} + +/* Slot rebuild after compaction: gids moved, so every surviving row must be + * re-hashed into a cleared slot array. Only partitions whose grp_count + * actually changed need it; an untouched partition's slots still describe its + * rows exactly. */ +typedef struct { + group_ht_t* part_hts; + const int8_t* key_types; + const uint8_t* dirty; /* [n_parts] */ +} topn_rebuild_ctx_t; + +static void topn_rebuild_fn(void* vctx, uint32_t worker_id, int64_t start, + int64_t end) { + (void)worker_id; + topn_rebuild_ctx_t* c = (topn_rebuild_ctx_t*)vctx; + for (int64_t p = start; p < end; p++) + if (c->dirty[p]) group_ht_rebuild_slots(&c->part_hts[p], c->key_types); +} + +/* (first_row, flat_id) pair for the sparse stable-order path below — + * sorting groups by earliest source row reproduces exactly the order the + * dense row-domain scan assigns. */ +typedef struct { + int64_t first; + uint32_t flat; +} group_order_pair_t; + +static int group_order_pair_cmp(const void* a, const void* b) { + int64_t fa = ((const group_order_pair_t*)a)->first; + int64_t fb = ((const group_order_pair_t*)b)->first; + return (fa > fb) - (fa < fb); +} + /* Selection-aware group iteration gate. When a WHERE leaves fewer than * nrows >> SEL_MATCH_GATE_SHIFT survivors, the high-card group build iterates * the survivor row list (match_idx) instead of scanning all nrows with a @@ -4678,6 +5229,12 @@ typedef struct { * "fewer than half the rows survive"; tuned in Task 2. */ #define SEL_MATCH_GATE_SHIFT 1 +/* Morsel geometry for radix_phase1_fn's column-major key staging: 256 rows + * x <= 8 keys = 16 KiB of worker stack, small enough to stay L1-resident while + * still amortising the per-key type dispatch over a useful batch. */ +#define P1_MORSEL 256u +#define P1_MORSEL_KEYS 8u + /* Per-worker, per-partition buffer of fat entries */ typedef struct { char* data; /* flat buffer: data[i * entry_stride] */ @@ -4693,11 +5250,12 @@ typedef struct { static inline void radix_buf_push(radix_buf_t* buf, uint16_t entry_stride, uint64_t hash, const int64_t* key_region_buf, const int64_t* agg_vals, uint16_t n_agg_vals, - int64_t row, uint16_t key_region) { + int64_t row, uint16_t key_region, + uint32_t cap0) { if (__builtin_expect(buf->count >= buf->cap, 0)) { uint32_t old_cap = buf->cap; if (old_cap > UINT32_MAX / 2) { buf->oom = true; return; } - uint32_t new_cap = old_cap ? old_cap * 2 : 1; + uint32_t new_cap = old_cap ? old_cap * 2 : (cap0 ? cap0 : 1); char* new_data = (char*)scratch_realloc( &buf->_hdr, (size_t)old_cap * entry_stride, (size_t)new_cap * entry_stride); @@ -4707,9 +5265,9 @@ static inline void radix_buf_push(radix_buf_t* buf, uint16_t entry_stride, } char* dst = buf->data + (size_t)buf->count * entry_stride; *(uint64_t*)dst = hash; - memcpy(dst + 8, key_region_buf, key_region); + ght_lanes_copy(dst + 8, key_region_buf, key_region); if (n_agg_vals) - memcpy(dst + 8 + key_region, agg_vals, (size_t)n_agg_vals * 8); + ght_lanes_copy(dst + 8 + key_region, agg_vals, (uint16_t)(n_agg_vals * 8)); memcpy(dst + entry_stride - 8, &row, 8); buf->count++; } @@ -4736,17 +5294,69 @@ typedef struct { /* When non-NULL, workers iterate match_idx[start..end) and * read row=match_idx[i]. When NULL, row=i. */ const int64_t* match_idx; + /* Expected rows per (worker,partition) — first-allocation capacity for + * the payload buffers. Growing 1->2->4->... instead re-copies the whole + * scattered payload ~2x (28% of ClickBench q16 in memmove). */ + uint32_t buf_prime; } radix_phase1_ctx_t; +/* Pack one row's aggregate input values into the entry staging slots. + * Shared by radix_phase1_fn's morsel-staged and row-major builders so the + * two cannot drift; slot indices come from the layout (agg_val_slot), never + * a running counter. */ +static inline void radix_phase1_pack_aggs(const radix_phase1_ctx_t* c, + const ght_layout_t* ly, + int64_t* agg_vals, int64_t row) { + const uint8_t* const aflags = ly->agg_flags; + uint16_t na = ly->n_aggs; + for (uint32_t a = 0; a < na; a++) { + uint8_t af = aflags[a]; + /* Destination slot from the layout (see group_rows_range): aggs + * with no value slot — holistic OP_MEDIAN &c. and valueless + * OP_COUNT — carry agg_val_slot == -1 and pack nothing. */ + int8_t vs = ly->agg_val_slot[a]; + if (vs < 0) continue; + uint8_t vi = (uint8_t)vs; + ray_t* ac = c->agg_vecs[a]; + if (!ac) continue; + if (c->agg_strlen && c->agg_strlen[a]) + agg_vals[vi] = group_strlen_at(ac, row); + else if (af & GHT_AF_F64) { + double v = group_fp_type(ac->type) + ? group_fp_at(ray_data(ac), ac->type, row) + : group_pack_i64_as_f64(ac, row); + memcpy(&agg_vals[vi], &v, sizeof(v)); + } + else + agg_vals[vi] = read_col_i64(ray_data(ac), row, ac->type, ac->attrs); + vi++; + /* Binary aggregator: read y-side value into the next slot. + * Cast non-F64 inputs through read_col_i64 — pearson_corr's + * finalize reads both slots as F64 doubles regardless of + * input type (i64 will be reinterpreted; for now we only + * support F64 inputs cleanly — i64 path is a perf followup). */ + if ((af & GHT_AF_BINARY) && c->agg_vecs2 && c->agg_vecs2[a]) { + ray_t* ay = c->agg_vecs2[a]; + if (af & GHT_AF_F64) { + double v = group_fp_type(ay->type) + ? group_fp_at(ray_data(ay), ay->type, row) + : group_pack_i64_as_f64(ay, row); + memcpy(&agg_vals[vi], &v, sizeof(v)); + } + else + agg_vals[vi] = group_pack_y_i64(ay, row); + vi++; + } + } +} + static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t end) { radix_phase1_ctx_t* c = (radix_phase1_ctx_t*)ctx; const ght_layout_t* ly = &c->layout; radix_buf_t* my_bufs = &c->bufs[(size_t)worker_id * c->n_parts]; uint16_t nk = ly->n_keys; - uint16_t na = ly->n_aggs; uint16_t nv = ly->n_agg_vals; const uint8_t* const kflags = ly->key_flags; - const uint8_t* const aflags = ly->agg_flags; uint8_t wide_any = ly->any_wide_key; uint8_t inline_str = ly->any_inline_str; uint16_t estride = ly->entry_stride; @@ -4774,6 +5384,66 @@ static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ } uint8_t nullable = c->nullable_mask; /* 0/1: any key may be null (see build) */ + uint8_t packed = ly->packed_key; + + /* ── Morsel-staged packed key build ────────────────────────────────────── + * read_col_i64 dispatches on the column type — and, for SYM, on the stored + * id width — so it lowers to a jump table: ONE INDIRECT JUMP per key per + * row. When a tuple mixes widths (ClickBench q18 is I64, I64, SYM32) the + * target alternates every key and the predictor cannot keep up; `notrack + * jmp` accounted for ~6% of the query inside this function. + * + * For packed, null-free layouts the keys are staged COLUMN-MAJOR over a + * morsel of rows first (group_keys_gather), so the dispatch happens once + * per key per MORSEL and each column is read strictly sequentially. The + * per-row half then only transposes lanes, hashes and pushes. + * + * Rows are visited in exactly the same order, the entry bytes and the + * hash are the same values the row-major loop below would produce, and the + * partition assignment follows from the hash — staging order only. */ + if (packed && !nullable && !inline_str && ly->null_words == 0 && + nk >= 1 && nk <= P1_MORSEL_KEYS) { + int64_t rowbuf[P1_MORSEL]; + static_assert(sizeof(int64_t) * P1_MORSEL * P1_MORSEL_KEYS <= 64u * 1024u, + "phase1 morsel stage must stay a modest worker-stack block"); + int64_t stage[P1_MORSEL_KEYS][P1_MORSEL]; + /* No match list and no row selection => the morsel's rows are exactly + * rowbuf[0] .. rowbuf[0]+m-1, so the columns can be read as runs. */ + bool contiguous = !match_idx && !c->rowsel; + for (int64_t i = start; i < end; ) { + /* Cancellation checkpoint once per morsel — strictly more + * responsive than the row-major loop's every-65536-rows poll. */ + if (ray_interrupted()) break; + uint32_t m = 0; + for (; i < end && m < P1_MORSEL; i++) { + int64_t row = match_idx ? match_idx[i] : i; + if (!match_idx && c->rowsel && !group_rowsel_pass(c->rowsel, row)) + continue; + rowbuf[m++] = row; + } + if (m == 0) continue; + for (uint32_t k = 0; k < nk; k++) { + if (contiguous) + group_keys_gather_seq(stage[k], c->key_data[k], rowbuf[0], m, + c->key_types[k], c->key_attrs[k]); + else + group_keys_gather(stage[k], c->key_data[k], rowbuf, m, + c->key_types[k], c->key_attrs[k]); + } + for (uint32_t j = 0; j < m; j++) { + int64_t row = rowbuf[j]; + for (uint32_t k = 0; k < nk; k++) keys[k] = stage[k][j]; + uint64_t h = ght_hash_lanes(keys, nk); + radix_phase1_pack_aggs(c, ly, agg_vals, row); + uint32_t part = group_radix_part(h, c->n_parts); + radix_buf_push(&my_bufs[part], estride, h, keys, + agg_vals, nv, row, ly->key_region, c->buf_prime); + } + } + scratch_free(stage_hdr); /* NULL (inline staging) → no-op */ + return; + } + for (int64_t i = start; i < end; i++) { /* Cancellation checkpoint every 65536 rows — ~150 polls on a * 10M-row ingest, imperceptible in the inner loop and still @@ -4792,6 +5462,25 @@ static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ int64_t* nullw = keys + nk; uint32_t null_words = ly->null_words; for (uint32_t w = 0; w < null_words; w++) nullw[w] = 0; + if (packed) { + /* Packed tuple (ly->packed_key): every key is a plain integer lane, + * so the wide/F64 arms of the generic loop below are dead and the + * whole key region — values plus null-mask words — avalanches in + * ONE ght_hash_lanes call instead of nk hashes plus nk-1 combines. + * The generic loop stays byte-for-byte what it was for every other + * key shape. */ + for (uint32_t k = 0; k < nk; k++) { + if (__builtin_expect(nullable && ray_key_may_be_null(c->key_vecs[k]) + && ray_vec_is_null(c->key_vecs[k], row), 0)) { + nullw[k >> 6] |= (int64_t)((uint64_t)1 << (k & 63)); + keys[k] = 0; + } else { + keys[k] = read_col_i64(c->key_data[k], row, c->key_types[k], + c->key_attrs[k]); + } + } + h = ght_hash_lanes(keys, (uint32_t)nk + null_words); + } else { for (uint32_t k = 0; k < nk; k++) { int8_t t = c->key_types[k]; uint64_t kh; @@ -4805,10 +5494,11 @@ static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ keys[k] = row; kh = wide_key_hash_at(ly, k, c->key_data, c->key_pool, row); } else if (t == RAY_F64) { - int64_t kv; - memcpy(&kv, &((double*)c->key_data[k])[row], 8); + /* -0.0 -> +0.0 before BOTH the store and the hash. */ + int64_t kv = group_key_f64_bits(c->key_data[k], row); + double dv; memcpy(&dv, &kv, 8); keys[k] = kv; - kh = ray_hash_f64(((double*)c->key_data[k])[row]); + kh = ray_hash_f64(dv); } else { int64_t kv = read_col_i64(c->key_data[k], row, t, c->key_attrs[k]); keys[k] = kv; @@ -4818,49 +5508,14 @@ static void radix_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ } h = ght_hash_null_words(h, nullw, null_words); } - - uint8_t vi = 0; - for (uint32_t a = 0; a < na; a++) { - uint8_t af = aflags[a]; - /* Holistic agg (OP_MEDIAN): no slot reserved — skip - * packing. Source column is read in the post-radix pass. */ - if (af & GHT_AF_HOLISTIC) continue; - ray_t* ac = c->agg_vecs[a]; - if (!ac) continue; - if (c->agg_strlen && c->agg_strlen[a]) - agg_vals[vi] = group_strlen_at(ac, row); - else if (af & GHT_AF_F64) { - double v = group_fp_type(ac->type) - ? group_fp_at(ray_data(ac), ac->type, row) - : group_pack_i64_as_f64(ac, row); - memcpy(&agg_vals[vi], &v, sizeof(v)); - } - else - agg_vals[vi] = read_col_i64(ray_data(ac), row, ac->type, ac->attrs); - vi++; - /* Binary aggregator: read y-side value into the next slot. - * Cast non-F64 inputs through read_col_i64 — pearson_corr's - * finalize reads both slots as F64 doubles regardless of - * input type (i64 will be reinterpreted; for now we only - * support F64 inputs cleanly — i64 path is a perf followup). */ - if ((af & GHT_AF_BINARY) && c->agg_vecs2 && c->agg_vecs2[a]) { - ray_t* ay = c->agg_vecs2[a]; - if (af & GHT_AF_F64) { - double v = group_fp_type(ay->type) - ? group_fp_at(ray_data(ay), ay->type, row) - : group_pack_i64_as_f64(ay, row); - memcpy(&agg_vals[vi], &v, sizeof(v)); - } - else - agg_vals[vi] = group_pack_y_i64(ay, row); - vi++; - } } + radix_phase1_pack_aggs(c, ly, agg_vals, row); + uint32_t part = group_radix_part(h, c->n_parts); radix_buf_push(&my_bufs[part], estride, h, inline_str ? (const int64_t*)keybuf : keys, - agg_vals, nv, row, ly->key_region); + agg_vals, nv, row, ly->key_region, c->buf_prime); } scratch_free(stage_hdr); /* NULL (inline staging) → no-op */ } @@ -4949,7 +5604,7 @@ static void radix_phase3_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ const char* row = ph->rows + (size_t)gi * rs; const char* rk = row + 8; /* key region (key_off-addressed) */ int64_t cnt = *(const int64_t*)(const void*)row; - const int64_t* nullw = (const int64_t*)(const void*)(rk + koff[nk]); + const int64_t* nullw = ght_null_words_at(ly, rk); /* Per-slot non-null count when nullable aggs are present; NULL * (→ use cnt) for null-free layouts (byte-identical to before). */ const int64_t* nnbase = ly->off_nn @@ -5016,7 +5671,9 @@ static void radix_phase3_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ /* nn = per-slot non-null count (nullable layout) or the group * row count (null-free). Drives the AVG/VAR/STDDEV divisor * and the all-null → typed-null decision, matching the DA path. */ - int64_t nn = nnbase ? nnbase[s] : cnt; + /* s < 0 for a valueless agg (OP_COUNT): it owns no off_nn + * slot, and its emit uses cnt. Never index nnbase with it. */ + int64_t nn = (nnbase && s >= 0) ? nnbase[s] : cnt; if (ao->out_type == RAY_F64) { double v; switch (op) { @@ -5161,6 +5818,7 @@ typedef struct { * Each partition HT stashes the ones matching wide_key_mask. */ void** key_data; const void** key_pool; /* [n_keys] str-pool base per wide STR key */ + topn_fuse_t fuse; /* .items NULL when top-k fusion is off */ } radix_phase2_ctx_t; static void radix_phase2_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t end) { @@ -5169,12 +5827,23 @@ static void radix_phase2_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ uint16_t estride = c->layout.entry_stride; for (int64_t p = start; p < end; p++) { + /* Before any `continue`: a partition that produces no HT produces no + * top-k candidates (and this run may be re-using a stash a bailed v2 + * attempt already wrote into). */ + if (c->fuse.items) c->fuse.item_cnt[p] = 0; uint32_t total = 0; for (uint32_t w = 0; w < c->n_workers; w++) total += c->bufs[(size_t)w * c->n_parts + p].count; if (total == 0) continue; - if (!group_ht_init_sized(&c->part_hts[p], 2, &c->layout, 1)) + /* `total` counts ROWS, an upper bound on groups — start from it + * (capped: heavy-duplicate partitions shouldn't over-allocate) so + * high-card partitions skip the 2->4->...->N rehash ladder. */ + uint32_t fe_cap = 2; + uint32_t fe_target = total < 4096 ? total : 4096; + while (fe_cap < fe_target) fe_cap <<= 1; + uint32_t fe_blk = total < 256 ? total : 256; + if (!group_ht_init_sized(&c->part_hts[p], fe_cap, &c->layout, fe_blk)) continue; /* Wide keys need source-column resolution during probe/rehash. */ if (c->layout.any_wide_key && c->key_data) { @@ -5188,6 +5857,11 @@ static void radix_phase2_fn(void* ctx, uint32_t worker_id, int64_t start, int64_ group_rows_indirect(&c->part_hts[p], c->key_types, buf->data, buf->count, estride); } + /* Partition complete (every worker's entries folded in, so the + * ordering agg's row slot is final) and still cache-hot: run its + * top-k here instead of re-reading the rows from DRAM later. */ + if (c->fuse.items) + topn_fuse_partition(&c->fuse, &c->part_hts[p], (uint32_t)p); } } @@ -5297,9 +5971,61 @@ typedef struct { ght_layout_t layout; ray_t* rowsel; const int64_t* match_idx; + uint32_t ht_init_cap; /* first-touch HT capacity: expected rows + * per (worker,partition), clamped to + * [2, 256]. Starting at 2 regardless of + * load re-rehashes every table ~8 times + * (ClickBench q32); empty pairs still + * allocate nothing. */ + uint32_t ht_init_block; /* row-block granularity for the same */ + uint32_t ht_grow_cap; /* slot-count ceiling for HT growth — + * see group_ht_t.grow_cap */ _Atomic(int) oom; } radix_v2_phase1_ctx_t; +/* Pack one row's aggregate input values into the entry's agg-value slots. + * Shared by radix_v2_phase1_fn's morsel-staged and row-major builders so the + * two cannot drift; slot indices come from the layout (agg_val_slot), never a + * running counter. Mirrors radix_phase1_pack_aggs, but tolerates a NULL + * agg_vecs table (pure-COUNT layouts carry none). */ +static inline void radix_v2_pack_aggs(const radix_v2_phase1_ctx_t* c, + const ght_layout_t* ly, + int64_t* ev, int64_t row) { + const uint8_t* const aflags = ly->agg_flags; + uint16_t na = ly->n_aggs; + for (uint32_t a = 0; a < na; a++) { + uint8_t af = aflags[a]; + int8_t vs = ly->agg_val_slot[a]; + if (vs < 0) continue; /* holistic / valueless OP_COUNT */ + uint8_t vi = (uint8_t)vs; + ray_t* ac = c->agg_vecs ? c->agg_vecs[a] : NULL; + if (!ac) continue; + if (c->agg_strlen && c->agg_strlen[a]) + ev[vi] = group_strlen_at(ac, row); + else if (af & GHT_AF_F64) { + double v = group_fp_type(ac->type) + ? group_fp_at(ray_data(ac), ac->type, row) + : group_pack_i64_as_f64(ac, row); + memcpy(&ev[vi], &v, sizeof(v)); + } + else + ev[vi] = read_col_i64(ray_data(ac), row, ac->type, ac->attrs); + vi++; + if ((af & GHT_AF_BINARY) && c->agg_vecs2 && c->agg_vecs2[a]) { + ray_t* ay = c->agg_vecs2[a]; + if (af & GHT_AF_F64) { + double v = group_fp_type(ay->type) + ? group_fp_at(ray_data(ay), ay->type, row) + : group_pack_i64_as_f64(ay, row); + memcpy(&ev[vi], &v, sizeof(v)); + } + else + ev[vi] = group_pack_y_i64(ay, row); + vi++; + } + } +} + static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t end) { radix_v2_phase1_ctx_t* c = (radix_v2_phase1_ctx_t*)ctx; @@ -5307,7 +6033,6 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, const ght_layout_t* ly = &c->layout; uint16_t nk = ly->n_keys; const uint8_t* const kflags = ly->key_flags; - const uint8_t* const aflags = ly->agg_flags; uint8_t wide_any = ly->any_wide_key; uint8_t inline_str = ly->any_inline_str; uint8_t nullable = c->nullable_mask; @@ -5338,6 +6063,187 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, kpool = (const void**)(blk + ly->entry_stride); } derive_key_pool(ly, c->key_vecs, kpool); + + /* Partition-major fast path (spec Part A): stage a morsel of rows + * (hash + partition, no HT access), counting-sort the morsel's + * indices by partition, then probe each partition's rows + * back-to-back against ONE hash table with slot-line prefetch. + * Within a partition run the HT struct, metadata, and slot array + * stay cache-hot, unlike the row-major order where consecutive + * rows hit different partitions. Hashing, entry layout, probe and + * merge are IDENTICAL to the generic loop below — iteration order + * only; group ids and first_row (MIN) are order-independent. + * + * Order-independence extends to the accumulators (COUNT/SUM/AVG, the + * only shapes the caller's v2 gate admits) because the counting sort is + * STABLE and a group lives entirely inside one partition: rows of the + * same group are visited in their original relative order, so even an + * f64 SUM accumulates in exactly the sequence the row-major loop uses. + * Only the interleaving BETWEEN partitions changes. + * + * NOTE: "hashing is identical" is load-bearing and is enforced by the + * v2_packed branch in the staging loop — see the comment there. */ + if (!wide_any && !inline_str && !nullable && ly->null_words == 0 && + (ly->need_flags & ~(uint32_t)GHT_NEED_SUM) == 0 && + !(ly->agg_flags_any & (GHT_AF_FIRST | GHT_AF_LAST | GHT_AF_BINARY | + GHT_AF_HOLISTIC)) && + nk >= 1 && nk <= 2 && ly->entry_stride <= 64) { + enum { V2M = 1024, V2MPF = 8 }; + /* Morsel staging: hashes, key values, source rows, partition ids, + * partition-order permutation, and the counting-sort histogram. + * mpart is uint32_t: group_radix_part_count is uncapped, so at + * extreme row counts (>4.29e9) n_parts can exceed 65536 and a + * uint16_t partition id would truncate and misroute the probe. + * morder stays uint16_t — it indexes morsel slots, always <=1024. + * hist is sized n_parts (not n_parts+1: sparse scheme below writes + * counts directly into hist[p], no running-offset slot needed) and + * is scratch_calloc'd ONCE per dispatch call so it starts zeroed; + * each morsel only touches (and only re-zeroes) the partitions it + * actually hit, tracked via plist, keeping the per-morsel counting + * sort O(morsel size) instead of O(n_parts) — at 100M rows + * n_parts can reach ~16384, and a full memset+prefix per 1024-row + * morsel would dominate the useful work. */ + uint8_t v2_packed = ly->packed_key; + uint64_t mh[V2M]; + int64_t mk[V2M * 2]; + int64_t mrow[V2M]; + uint32_t mpart[V2M]; + uint16_t morder[V2M]; + uint32_t plist[V2M]; + ray_t* hist_hdr = NULL; + uint32_t* hist = (uint32_t*)scratch_calloc(&hist_hdr, + (size_t)c->n_parts * sizeof(uint32_t)); + if (!hist) { + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + scratch_free(v2_stage_hdr); + return; + } + + int64_t i = start; + while (i < end) { + if (ray_interrupted()) break; + /* ---- stage ---- */ + uint32_t mn = 0; + for (; i < end && mn < V2M; i++) { + int64_t row = match_idx ? match_idx[i] : i; + if (!match_idx && c->rowsel && + !group_rowsel_pass(c->rowsel, row)) + continue; + uint64_t h = 0; + for (uint32_t k = 0; k < nk; k++) { + uint64_t kh = 0; + int8_t t = c->key_types[k]; + if (t == RAY_F64) { + /* -0.0 -> +0.0 before BOTH the store and the hash. + * (v2_packed never holds with an F64 key — F64 is off + * the packed whitelist — but stage the same bits the + * generic builder would either way.) */ + int64_t kv = group_key_f64_bits(c->key_data[k], row); + double dv; memcpy(&dv, &kv, 8); + mk[(size_t)mn * nk + k] = kv; + if (!v2_packed) kh = ray_hash_f64(dv); + } else { + int64_t kv = read_col_i64(c->key_data[k], row, t, + c->key_attrs[k]); + mk[(size_t)mn * nk + k] = kv; + if (!v2_packed) kh = ray_hash_i64(kv); + } + if (!v2_packed) h = (k == 0) ? kh : ray_hash_combine(h, kh); + } + /* Packed layouts MUST hash through ght_hash_lanes here, not + * per-key hash + combine. hash_keys_inline — which + * group_ht_rehash, group_ht_rebuild_slots and group_merge_row + * all use to re-derive a row's hash — returns ght_hash_lanes + * for this layout, so a per-key hash staged here would stop + * matching the moment a worker HT rehashes: the probe would + * land on the wrong slot chain and insert a DUPLICATE group + * row for a key already present. (Phase 2's merge folds the + * duplicates so answers stayed right, which is exactly why it + * would have gone unnoticed.) This whole fast path was + * unreachable before null-mask elision made null_words == 0 + * satisfiable, so the divergence arrived with it. + * null_words == 0 is in the gate above, hence lanes == nk. */ + if (v2_packed) h = ght_hash_lanes(&mk[(size_t)mn * nk], nk); + mh[mn] = h; + mrow[mn] = row; + mpart[mn] = group_radix_part(h, c->n_parts); + mn++; + } + if (mn == 0) continue; + + /* ---- counting-sort morsel indices by partition, touched-set + * only ---- */ + uint32_t np = 0; + for (uint32_t j = 0; j < mn; j++) { + uint32_t p = mpart[j]; + if (hist[p] == 0) plist[np++] = p; + hist[p]++; + } + uint32_t run = 0; + for (uint32_t t = 0; t < np; t++) { + uint32_t p = plist[t]; + uint32_t cnt = hist[p]; + hist[p] = run; + run += cnt; + } + for (uint32_t j = 0; j < mn; j++) + morder[hist[mpart[j]]++] = (uint16_t)j; + /* Restore the all-zero invariant for the next morsel — only + * the touched entries need clearing. */ + for (uint32_t t = 0; t < np; t++) hist[plist[t]] = 0; + + /* ---- probe, partition-major with intra-run prefetch ---- */ + for (uint32_t o = 0; o < mn; o++) { + uint32_t j = morder[o]; + uint32_t p = mpart[j]; + if (!my_hts[p].slots) { + if (!group_ht_init_sized(&my_hts[p], c->ht_init_cap, ly, + c->ht_init_block)) { + atomic_store_explicit(&c->oom, 1, + memory_order_relaxed); + goto v2_morsel_done; + } + my_hts[p].grow_cap = c->ht_grow_cap; + masks[p] = my_hts[p].ht_cap - 1; + } + /* Prefetch the slot line V2MPF entries ahead WITHIN this + * morsel — mostly the same partition, so the mask read is + * usually exact and the prefetch always harmless. */ + if (o + V2MPF < mn) { + uint32_t jn = morder[o + V2MPF]; + uint32_t pn = mpart[jn]; + if (my_hts[pn].slots) + __builtin_prefetch( + &my_hts[pn].slots[(uint32_t)(mh[jn] & masks[pn])], + 1, 1); + } + /* Build the entry in ebuf exactly as the generic loop + * does: [hash][keys][row]. */ + *(uint64_t*)ebuf = mh[j]; + int64_t* ek = (int64_t*)(ebuf + 8); + for (uint32_t k = 0; k < nk; k++) + ek[k] = mk[(size_t)j * nk + k]; + /* Agg inputs are read here, not staged, so the morsel arrays + * stay the same size; the source row is mrow[j] and the reads + * are identical to the row-major builder's. */ + if (ly->need_flags) + radix_v2_pack_aggs(c, ly, + (int64_t*)(ebuf + 8 + (size_t)ly->key_region), mrow[j]); + memcpy(ebuf + ly->entry_stride - 8, &mrow[j], 8); + masks[p] = group_probe_entry(&my_hts[p], ebuf, + c->key_types, masks[p]); + if (my_hts[p].oom) { + atomic_store_explicit(&c->oom, 1, memory_order_relaxed); + goto v2_morsel_done; + } + } + } +v2_morsel_done: + scratch_free(hist_hdr); + scratch_free(v2_stage_hdr); + return; + } + for (int64_t i = start; i < end; i++) { if (((i - start) & 65535) == 0 && ray_interrupted()) break; int64_t row = match_idx ? match_idx[i] : i; @@ -5352,6 +6258,23 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, int64_t* nullw = ek + nk; /* null-mask words at key_off[nk]==nk*8 */ uint32_t null_words = ly->null_words; for (uint32_t w = 0; w < null_words; w++) nullw[w] = 0; + if (ly->packed_key) { + /* Packed tuple (ly->packed_key): every key is a plain integer lane, so + * the wide/F64 arms of the generic loop below are dead and the whole + * key region — values plus null-mask words — avalanches in ONE + * ght_hash_lanes call instead of nk hashes plus nk-1 combines. MUST + * stay in lockstep with hash_keys_inline (rehash/merge/lookup). */ + for (uint32_t k = 0; k < nk; k++) { + if (__builtin_expect(nullable && ray_key_may_be_null(c->key_vecs[k]) + && ray_vec_is_null(c->key_vecs[k], row), 0)) { + nullw[k >> 6] |= (int64_t)((uint64_t)1 << (k & 63)); + ek[k] = 0; + } else { + ek[k] = read_col_i64(c->key_data[k], row, c->key_types[k], c->key_attrs[k]); + } + } + h = ght_hash_lanes(ek, (uint32_t)nk + null_words); + } else { for (uint32_t k = 0; k < nk; k++) { int8_t t = c->key_types[k]; uint64_t kh; @@ -5365,10 +6288,11 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, ek[k] = row; kh = wide_key_hash_at(ly, k, c->key_data, kpool, row); } else if (t == RAY_F64) { - int64_t kv; - memcpy(&kv, &((double*)c->key_data[k])[row], 8); + /* -0.0 -> +0.0 before BOTH the store and the hash. */ + int64_t kv = group_key_f64_bits(c->key_data[k], row); + double dv; memcpy(&dv, &kv, 8); ek[k] = kv; - kh = ray_hash_f64(((double*)c->key_data[k])[row]); + kh = ray_hash_f64(dv); } else { int64_t kv = read_col_i64(c->key_data[k], row, t, c->key_attrs[k]); ek[k] = kv; @@ -5378,55 +6302,27 @@ static void radix_v2_phase1_fn(void* ctx, uint32_t worker_id, } h = ght_hash_null_words(h, nullw, null_words); } + } *(uint64_t*)ebuf = h; /* Pack agg values into entry — only when the HT layout actually * reads them. For count-only need_flags == 0 and accum_from_entry * skips every agg slot; packing here would be a wasted column * read per row (a measurable regression on q15-class queries). */ - if (ly->need_flags) { - int64_t* ev = (int64_t*)(ebuf + 8 + (size_t)ly->key_region); - uint8_t vi = 0; - uint16_t na = ly->n_aggs; - for (uint32_t a = 0; a < na; a++) { - uint8_t af = aflags[a]; - if (af & GHT_AF_HOLISTIC) continue; - ray_t* ac = c->agg_vecs ? c->agg_vecs[a] : NULL; - if (!ac) continue; - if (c->agg_strlen && c->agg_strlen[a]) - ev[vi] = group_strlen_at(ac, row); - else if (af & GHT_AF_F64) { - double v = group_fp_type(ac->type) - ? group_fp_at(ray_data(ac), ac->type, row) - : group_pack_i64_as_f64(ac, row); - memcpy(&ev[vi], &v, sizeof(v)); - } - else - ev[vi] = read_col_i64(ray_data(ac), row, ac->type, ac->attrs); - vi++; - if ((af & GHT_AF_BINARY) && c->agg_vecs2 && c->agg_vecs2[a]) { - ray_t* ay = c->agg_vecs2[a]; - if (af & GHT_AF_F64) { - double v = group_fp_type(ay->type) - ? group_fp_at(ray_data(ay), ay->type, row) - : group_pack_i64_as_f64(ay, row); - memcpy(&ev[vi], &v, sizeof(v)); - } - else - ev[vi] = group_pack_y_i64(ay, row); - vi++; - } - } - } + if (ly->need_flags) + radix_v2_pack_aggs(c, ly, + (int64_t*)(ebuf + 8 + (size_t)ly->key_region), row); memcpy(ebuf + ly->entry_stride - 8, &row, 8); uint32_t p = group_radix_part(h, c->n_parts); if (!my_hts[p].slots) { /* Demand-driven initialization: the first observed row creates * the smallest valid table and normal growth follows actual * cardinality. Empty worker/partition pairs allocate nothing. */ - if (!group_ht_init_sized(&my_hts[p], 2, ly, 1)) { + if (!group_ht_init_sized(&my_hts[p], c->ht_init_cap, ly, + c->ht_init_block)) { atomic_store_explicit(&c->oom, 1, memory_order_relaxed); break; } + my_hts[p].grow_cap = c->ht_grow_cap; if (wide_any && c->key_data) { group_ht_set_key_data(&my_hts[p], c->key_data); group_ht_set_key_pool(&my_hts[p], c->key_vecs); @@ -5453,6 +6349,7 @@ typedef struct { ght_layout_t layout; void** key_data; const void** key_pool; /* [n_keys] str-pool base per wide STR key */ + topn_fuse_t fuse; /* .items NULL when top-k fusion is off */ _Atomic(int) oom; } radix_v2_phase2_ctx_t; @@ -5463,6 +6360,8 @@ static void radix_v2_phase2_fn(void* ctx, uint32_t worker_id, if (atomic_load_explicit(&c->oom, memory_order_relaxed)) return; uint16_t row_stride = c->layout.row_stride; for (int64_t p = start; p < end; p++) { + /* Cleared before any `continue`/`return` below — see topn_fuse_t. */ + if (c->fuse.items) c->fuse.item_cnt[p] = 0; /* Upper bound on the merged partition: sum of worker grp_counts * (some keys may be present in multiple workers — the merge will * fold those, so the final grp_count is ≤ this sum). */ @@ -5470,7 +6369,14 @@ static void radix_v2_phase2_fn(void* ctx, uint32_t worker_id, for (uint32_t w = 0; w < c->n_workers; w++) total_grps += c->wpart_hts[(size_t)w * c->n_parts + p].grp_count; if (total_grps == 0) continue; - if (!group_ht_init_sized(&c->part_hts[p], 2, &c->layout, 1)) { + /* Size the merged table from the known upper bound instead of + * growing 2->4->...->N — with thousands of groups per partition the + * repeated rehash+re-insert dominated the merge (ClickBench q32). */ + uint32_t merge_cap = 2; + while (merge_cap < total_grps && merge_cap < (1u << 30)) merge_cap <<= 1; + uint32_t merge_blk = total_grps < 256 ? total_grps : 256; + if (!group_ht_init_sized(&c->part_hts[p], merge_cap, &c->layout, + merge_blk)) { atomic_store_explicit(&c->oom, 1, memory_order_relaxed); return; } @@ -5493,6 +6399,10 @@ static void radix_v2_phase2_fn(void* ctx, uint32_t worker_id, } } } + /* Merge for this partition is complete (count/sum slots final) and + * the rows are cache-hot — same fused top-k as the fat-entry path. */ + if (c->fuse.items) + topn_fuse_partition(&c->fuse, &c->part_hts[p], (uint32_t)p); } } @@ -6149,17 +7059,29 @@ static bool sparse_i64_touch(sparse_i64_ht_t* ht, int64_t key, uint8_t n_aggs, return true; } +/* Sign discipline for DA key reads: the min/max prescan (minmax_scan_fn) + * reads narrow keys through their SIGNED column type, so key_mins[k] is a + * sign-extended value. Every accumulate-side read must sign-extend the + * same way or a negative I16/I32/DATE/TIME key maps to a slot far outside + * [0, range) — an out-of-bounds write, not just a wrong group. Only + * BOOL/U8 and SYM dictionary ids are unsigned. */ +static inline int da_key_is_unsigned(int8_t t) { + return t == RAY_BOOL || t == RAY_U8 || RAY_IS_SYM(t); +} + /* Composite GID from multi-key. Planning bounds total_slots to INT32_MAX. */ static inline int32_t da_composite_gid(da_ctx_t* c, int64_t r) { int32_t gid = 0; for (uint32_t k = 0; k < c->n_keys; k++) { - int64_t val = read_by_esz(c->key_ptrs[k], r, c->key_esz[k]); + int64_t val = read_signed_by_esz(c->key_ptrs[k], r, c->key_esz[k], + da_key_is_unsigned(c->key_types[k])); gid += (int32_t)((val - c->key_mins[k]) * c->key_strides[k]); } return gid; } -/* Typed composite GID: eliminates per-element switch when all keys share width */ +/* Typed composite GID: eliminates per-element switch when all keys share + * width AND signedness (see da_key_is_unsigned above). */ #define DEFINE_DA_COMPOSITE_GID_TYPED(SUFFIX, KTYPE) \ static inline int32_t da_composite_gid_##SUFFIX(da_ctx_t* c, int64_t r) { \ int32_t gid = 0; \ @@ -6172,6 +7094,8 @@ static inline int32_t da_composite_gid_##SUFFIX(da_ctx_t* c, int64_t r) { \ DEFINE_DA_COMPOSITE_GID_TYPED(u8, uint8_t) DEFINE_DA_COMPOSITE_GID_TYPED(u16, uint16_t) DEFINE_DA_COMPOSITE_GID_TYPED(u32, uint32_t) +DEFINE_DA_COMPOSITE_GID_TYPED(i16, int16_t) +DEFINE_DA_COMPOSITE_GID_TYPED(i32, int32_t) DEFINE_DA_COMPOSITE_GID_TYPED(i64, int64_t) #undef DEFINE_DA_COMPOSITE_GID_TYPED @@ -6826,11 +7750,23 @@ static void da_accum_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t en } while (0) if (n_keys == 1) { - switch (c->key_esz[0]) { - case 1: DA_SINGLE_KEY_LOOP(uint8_t, ); break; - case 2: DA_SINGLE_KEY_LOOP(uint16_t, ); break; - case 4: DA_SINGLE_KEY_LOOP(uint32_t, (int64_t)); break; - default: DA_SINGLE_KEY_LOOP(int64_t, ); break; + /* Signed narrow keys (I16/I32/DATE/TIME) must sign-extend exactly + * as the min/max prescan did — an unsigned read maps a negative + * key to a slot outside [0, range) (see da_key_is_unsigned). */ + if (da_key_is_unsigned(c->key_types[0])) { + switch (c->key_esz[0]) { + case 1: DA_SINGLE_KEY_LOOP(uint8_t, ); break; + case 2: DA_SINGLE_KEY_LOOP(uint16_t, ); break; + case 4: DA_SINGLE_KEY_LOOP(uint32_t, (int64_t)); break; + default: DA_SINGLE_KEY_LOOP(int64_t, ); break; + } + } else { + switch (c->key_esz[0]) { + case 1: DA_SINGLE_KEY_LOOP(int8_t, ); break; + case 2: DA_SINGLE_KEY_LOOP(int16_t, ); break; + case 4: DA_SINGLE_KEY_LOOP(int32_t, (int64_t)); break; + default: DA_SINGLE_KEY_LOOP(int64_t, ); break; + } } #undef DA_SINGLE_KEY_LOOP return; @@ -6854,27 +7790,51 @@ static void da_accum_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t en } \ } while (0) - /* Check if all keys share the same element size */ + /* Typed loops need all keys to share element size AND signedness; a + * mixed set (e.g. I16 + SYM16) falls to the sign-aware generic gid. */ bool uniform_esz = true; - for (uint32_t k = 1; k < n_keys; k++) + bool uniform_sign = true; + for (uint32_t k = 1; k < n_keys; k++) { if (c->key_esz[k] != c->key_esz[0]) { uniform_esz = false; break; } + if (da_key_is_unsigned(c->key_types[k]) != + da_key_is_unsigned(c->key_types[0])) { uniform_sign = false; break; } + } - if (uniform_esz) { + if (uniform_esz && uniform_sign) { + bool k_uns = da_key_is_unsigned(c->key_types[0]); switch (c->key_esz[0]) { case 1: + if (k_uns) { #define GID_FN(R) da_composite_gid_u8(c, (R)) - DA_MULTI_KEY_LOOP(GID_FN); + DA_MULTI_KEY_LOOP(GID_FN); #undef GID_FN + } else { +#define GID_FN(R) da_composite_gid(c, (R)) + DA_MULTI_KEY_LOOP(GID_FN); +#undef GID_FN + } break; case 2: + if (k_uns) { #define GID_FN(R) da_composite_gid_u16(c, (R)) - DA_MULTI_KEY_LOOP(GID_FN); + DA_MULTI_KEY_LOOP(GID_FN); +#undef GID_FN + } else { +#define GID_FN(R) da_composite_gid_i16(c, (R)) + DA_MULTI_KEY_LOOP(GID_FN); #undef GID_FN + } break; case 4: + if (k_uns) { #define GID_FN(R) da_composite_gid_u32(c, (R)) - DA_MULTI_KEY_LOOP(GID_FN); + DA_MULTI_KEY_LOOP(GID_FN); #undef GID_FN + } else { +#define GID_FN(R) da_composite_gid_i32(c, (R)) + DA_MULTI_KEY_LOOP(GID_FN); +#undef GID_FN + } break; default: #define GID_FN(R) da_composite_gid_i64(c, (R)) @@ -7090,6 +8050,23 @@ static inline int64_t reprobe_flat_gid(const reprobe_ctx_t* c, int64_t row, int64_t* nullw = ek_buf + nk; uint32_t null_words = c->layout->null_words; for (uint32_t w = 0; w < null_words; w++) nullw[w] = 0; + if (c->layout->packed_key) { + /* Packed tuple (ly->packed_key): every key is a plain integer lane, so + * the wide/F64 arms of the generic loop below are dead and the whole + * key region — values plus null-mask words — avalanches in ONE + * ght_hash_lanes call instead of nk hashes plus nk-1 combines. MUST + * stay in lockstep with hash_keys_inline (rehash/merge/lookup). */ + for (uint32_t k = 0; k < nk; k++) { + if (__builtin_expect(nullable && ray_key_may_be_null(key_vecs[k]) + && ray_vec_is_null(key_vecs[k], row), 0)) { + nullw[k >> 6] |= (int64_t)((uint64_t)1 << (k & 63)); + ek_buf[k] = 0; + } else { + ek_buf[k] = read_col_i64(key_data[k], row, key_types[k], key_attrs[k]); + } + } + h = ght_hash_lanes(ek_buf, (uint32_t)nk + null_words); + } else { for (uint32_t k = 0; k < nk; k++) { int8_t t = key_types[k]; uint64_t kh; @@ -7103,10 +8080,12 @@ static inline int64_t reprobe_flat_gid(const reprobe_ctx_t* c, int64_t row, ek_buf[k] = row; kh = wide_key_hash_at(c->layout, k, key_data, c->key_pool, row); } else if (t == RAY_F64) { - int64_t kv; - memcpy(&kv, &((double*)key_data[k])[row], 8); + /* -0.0 -> +0.0 before BOTH the store and the hash — the + * reprobe MUST derive the same bits phase 1 stored. */ + int64_t kv = group_key_f64_bits(key_data[k], row); + double dv; memcpy(&dv, &kv, 8); ek_buf[k] = kv; - kh = ray_hash_f64(((double*)key_data[k])[row]); + kh = ray_hash_f64(dv); } else { int64_t kv = read_col_i64(key_data[k], row, t, key_attrs[k]); ek_buf[k] = kv; @@ -7115,6 +8094,7 @@ static inline int64_t reprobe_flat_gid(const reprobe_ctx_t* c, int64_t row, h = (k == 0) ? kh : ray_hash_combine(h, kh); } h = ght_hash_null_words(h, nullw, null_words); + } lookup_keys = ek_buf; } @@ -7816,6 +8796,96 @@ static ray_t* exec_group_parted(ray_graph_t* g, ray_op_t* op, ray_t* parted_tbl, static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t group_limit); +/* Trim a full group result to the emit filter's keep set: rows whose + * filtered-agg value passes min_count_exclusive and, when top_count_take + * is set, lies within the top-N by that value (ties INCLUDED — the result + * is a superset of N rows; the DAG's sort+take downstream finalizes the + * exact order/limit, exactly as it would on an untrimmed result). + * Row order is preserved. Consumes `result`, returns an owned table. */ +static ray_t* group_emit_filter_trim(ray_t* result, uint32_t n_keys, + uint32_t n_aggs, + ray_group_emit_filter_t ef) { + if (!result || RAY_IS_ERR(result) || result->type != RAY_TABLE) + return result; + if (ef.agg_index >= n_aggs) return result; + int64_t nrows = ray_table_nrows(result); + if (nrows <= 0) return result; + ray_t* vcol = ray_table_get_col_idx(result, + (int64_t)n_keys + ef.agg_index); + if (!vcol || (vcol->type != RAY_I64 && vcol->type != RAY_F64)) + return result; + bool is_f64 = (vcol->type == RAY_F64); + /* Direction comes straight from .desc (mirrors the v2_emit topn path): + * every arming site sets it, COUNT included. Coercing COUNT to + * largest-first here made `asc: take: N` keep the largest N + * (issue #408). */ + const int64_t* vi = (const int64_t*)ray_data(vcol); + const double* vf = (const double*)ray_data(vcol); + #define EF_VAL_D(r) (is_f64 ? vf[(r)] : (double)vi[(r)]) + + double thr = 0.0; + bool have_thr = false; + if (ef.top_count_take > 0 && nrows > ef.top_count_take) { + /* Quickselect (on a copy) for the N-th value in the keep + * direction: desc keeps the N largest -> threshold is the + * (nrows-N)-th ascending element; asc keeps the N smallest. */ + ray_t* sel_hdr = NULL; + double* sv = (double*)scratch_alloc(&sel_hdr, + (size_t)nrows * sizeof(double)); + if (sv) { + for (int64_t r = 0; r < nrows; r++) sv[r] = EF_VAL_D(r); + int64_t k = ef.desc ? (nrows - ef.top_count_take) + : (ef.top_count_take - 1); + int64_t lo = 0, hi = nrows - 1; + while (lo < hi) { + double pivot = sv[k]; + int64_t i = lo, j = hi; + while (i <= j) { + while (sv[i] < pivot) i++; + while (sv[j] > pivot) j--; + if (i <= j) { + double t = sv[i]; sv[i] = sv[j]; sv[j] = t; + i++; j--; + } + } + if (k <= j) hi = j; + else if (k >= i) lo = i; + else break; + } + thr = sv[k]; + have_thr = true; + scratch_free(sel_hdr); + } + } + + ray_t* idx = ray_vec_new(RAY_I64, nrows); + if (!idx || RAY_IS_ERR(idx)) { + if (idx) ray_error_free(idx); + return result; /* trim is an optimization — full result is valid */ + } + int64_t* ix = (int64_t*)ray_data(idx); + int64_t kept = 0; + for (int64_t r = 0; r < nrows; r++) { + double v = EF_VAL_D(r); + if (ef.min_count_exclusive > 0 && !(v > (double)ef.min_count_exclusive)) + continue; + if (have_thr && (ef.desc ? (v < thr) : (v > thr))) + continue; + ix[kept++] = r; + } + #undef EF_VAL_D + idx->len = kept; + if (kept == nrows) { ray_release(idx); return result; } + ray_t* out = ray_at_fn(result, idx); + ray_release(idx); + if (!out || RAY_IS_ERR(out)) { + if (out) ray_error_free(out); + return result; + } + ray_release(result); + return out; +} + /* Map an I32 dictionary-code result column back to strings via the source * column: code -> first_occ[code] -> the string at that row. */ static ray_t* dict_codes_to_str(const ray_t* codes_col, ray_t* src_col, @@ -8373,7 +9443,11 @@ static bool sg_shape_eligible(ray_graph_t* g, ray_op_t* op, ray_t* tbl, int64_t group_limit, ray_t** agg_vecs, ray_t** agg_vecs2, agg_prod_t* prod) { - if (group_limit != 0) return false; + /* group_limit is a HEAD(GROUP) HINT, not a semantic: this kernel computes + * every group and the caller's HEAD trims the result, so a positive limit + * is simply ignored here (staying eligible keeps where+by+take shapes on + * the slice kernel instead of dropping them to the generic ladder). */ + if (group_limit < 0) return false; if (ray_group_emit_filter_get().enabled) return false; ray_op_ext_t* ext = find_ext(g, op->id); if (!ext || ext->n_keys != 1 || ext->n_aggs < 1 || ext->n_aggs > 16) @@ -8661,7 +9735,8 @@ static ray_t* exec_group_slices(ray_graph_t* g, ray_op_t* op, ray_t* tbl, /* v2 bridge for expression agg inputs — see the call site in * exec_group_run. Returns the v2 result (or an error), or NULL when the * shape is ineligible and the caller should continue on the legacy path. */ -static ray_t* exec_group_v2_exprs(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { +static ray_t* exec_group_v2_exprs(ray_graph_t* g, ray_op_t* op, ray_t* tbl, + int64_t group_limit) { if (!tbl || tbl->type != RAY_TABLE) return NULL; ray_op_ext_t* ext = find_ext(g, op->id); /* Own gate: this v2-expression shadow path serves any key/agg count @@ -8770,7 +9845,7 @@ static ray_t* exec_group_v2_exprs(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { } ray_t* result = NULL; if (ok && agg_v2_can_handle(g, op2, sub)) - result = exec_group_v2(g, op2, sub); + result = exec_group_v2(g, op2, sub, group_limit); for (uint32_t a = 0; a < na; a++) if (synth[a]) ray_release(synth[a]); if (sub) ray_release(sub); @@ -8820,6 +9895,46 @@ typedef struct { ray_group_emit_filter_t emit_filter; } sp_dyn_ctx_t; +/* Parallel dense-count fill for the sp_dyn pure-count case: each worker + * scatters its row slice into a private uint32[bound] array; the caller + * merges them into range_count. The serial scatter was the Amdahl wall of + * every single-SYM-key "top-N by count" query (ClickBench q13: 88ms of a + * 110ms query in one thread's cache-miss loop; flat 8->16 core scaling). + * A key outside [0, bound) sets `fail` and the caller falls back to the + * serial path — correctness never depends on the bound being right. */ +typedef struct { + const void* key_data; + uint8_t key_esz; + uint32_t* counts; /* [n_workers * bound] */ + uint64_t bound; + uint32_t n_workers; + const int64_t* match_idx; + ray_t* rowsel; + _Atomic(int) fail; +} sp_dyn_pcount_ctx_t; + +static void sp_dyn_pcount_fn(void* vctx, uint32_t wid, int64_t start, + int64_t end) { + sp_dyn_pcount_ctx_t* c = (sp_dyn_pcount_ctx_t*)vctx; + if (atomic_load_explicit(&c->fail, memory_order_relaxed)) return; + uint32_t* my = c->counts + (size_t)(wid % c->n_workers) * c->bound; + const int64_t* match_idx = c->match_idx; + const uint64_t bound = c->bound; + const uint8_t esz = c->key_esz; + const void* kd = c->key_data; + for (int64_t i = start; i < end; i++) { + int64_t r = match_idx ? match_idx[i] : i; + if (!match_idx && c->rowsel && !group_rowsel_pass(c->rowsel, r)) + continue; + int64_t key = read_by_esz(kd, r, esz); + if ((uint64_t)key >= bound) { /* negative wraps huge — caught too */ + atomic_store_explicit(&c->fail, 1, memory_order_relaxed); + return; + } + if (my[key] != UINT32_MAX) my[key]++; + } +} + static ray_t* __attribute__((noinline)) exec_group_sp_dyn_emit(const sp_dyn_ctx_t* c) { ray_graph_t* g = c->g; @@ -8868,6 +9983,83 @@ exec_group_sp_dyn_emit(const sp_dyn_ctx_t* c) { uint64_t max_seen = 0; bool have_dyn_key = false; + + /* Parallel fill (pure-count only: range_sum == NULL). The + * bound must be known up front: SYM codes are bounded by + * their domain count, narrow keys by their width. Success + * skips the serial scatter below; any bound violation or + * allocation miss falls straight back to it. */ + bool dyn_par_done = false; + /* rowsel excluded: the serial loop below iterates the + * selection segment-aware (whole SEL_NONE morsels skipped), + * which beats a parallel per-row bitmap test on sparse + * selections (ClickBench q07: 2ms serial vs 6ms parallel). */ + if (dyn_ok && !range_sum && !rowsel && n_scan >= 262144) { + uint64_t bound = 0; + if (key_types[0] == RAY_SYM) { + int64_t dc = ray_sym_domain_count( + ray_sym_vec_domain(key_vecs[0])); + if (dc > 0 && (uint64_t)dc <= max_dense_cap) + bound = (uint64_t)dc; + } else if (key_esz == 1) bound = 256u; + else if (key_esz == 2) bound = 1u << 16; + ray_pool_t* dp = ray_pool_get(); + uint32_t dnw = dp ? ray_pool_total_workers(dp) : 1; + if (bound > 0 && dp && dnw >= 2 && + (uint64_t)dnw * bound * sizeof(uint32_t) <= (512u << 20)) { + ray_t* pc_hdr = NULL; + uint32_t* pc = (uint32_t*)scratch_calloc(&pc_hdr, + (size_t)dnw * bound * sizeof(uint32_t)); + if (pc) { + sp_dyn_pcount_ctx_t pctx = { + .key_data = key_data[0], + .key_esz = key_esz, + .counts = pc, + .bound = bound, + .n_workers = dnw, + .match_idx = match_idx, + .rowsel = rowsel, + .fail = 0, + }; + ray_pool_dispatch(dp, sp_dyn_pcount_fn, &pctx, + n_scan); + if (!atomic_load_explicit(&pctx.fail, + memory_order_relaxed)) { + /* Grow range_count to the bound, then merge + * worker arrays with saturation. */ + if (bound > cap) { + uint32_t* nc = (uint32_t*)scratch_realloc( + &cnt_hdr, (size_t)cap * sizeof(uint32_t), + (size_t)bound * sizeof(uint32_t)); + if (nc) { + range_count = nc; + memset(range_count + cap, 0, + (size_t)(bound - cap) * sizeof(uint32_t)); + cap = bound; + } + } + if (bound <= cap) { + for (uint32_t w = 0; w < dnw; w++) { + const uint32_t* src = pc + (size_t)w * bound; + for (uint64_t o = 0; o < bound; o++) { + uint64_t s = (uint64_t)range_count[o] + src[o]; + range_count[o] = s > UINT32_MAX + ? UINT32_MAX : (uint32_t)s; + } + } + for (uint64_t o = 0; o < bound; o++) + if (range_count[o] > 0) { + have_dyn_key = true; + max_seen = o; + } + dyn_par_done = true; + } + } + scratch_free(pc_hdr); + } + } + } + #define DYN_DENSE_ACCUM_ROW(row_expr) \ do { \ int64_t dyn_row = (row_expr); \ @@ -8922,7 +10114,9 @@ exec_group_sp_dyn_emit(const sp_dyn_ctx_t* c) { } \ } while (0) - if (dyn_ok && match_idx) { + if (dyn_par_done) { + /* counts merged by the parallel fill above */ + } else if (dyn_ok && match_idx) { for (int64_t i = 0; i < n_scan; i++) DYN_DENSE_ACCUM_ROW(match_idx[i]); } else if (dyn_ok && rowsel) { @@ -9161,10 +10355,61 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, /* v2 doesn't implement the top-count emit filter (old-engine feature); * when one is active, stay on the legacy path that honors it. */ - if (ray_agg_engine_v2 && group_limit == 0 + /* group_limit is a HINT (HEAD(GROUP) fusion), not a semantic: v2 threads + * it down to the radix strategy's bounded emit and the caller trims the + * result either way, so a positive limit stays on v2 rather than falling + * back to the (slower, full-materialization) legacy ladder. */ + if (ray_agg_engine_v2 && group_limit >= 0 && !ray_group_emit_filter_get().enabled && agg_v2_can_handle(g, op, tbl)) - return exec_group_v2(g, op, tbl); + return exec_group_v2(g, op, tbl, group_limit); + + /* Emit-filter shape on a wide-domain SYM key: the sp dense/sparse + * ladder below is single-threaded and its dense array scales with the + * store's SHARED sym domain (splayed stores keep one domain across all + * SYM columns — often 10M+ ids), so the scatter becomes the query's + * serial wall (ClickBench q13: 88ms of a 110ms query, flat multi-core + * scaling). Run the PARALLEL v2 engine instead and trim its full + * result to the filter's top-N superset — the emit filter is purely an + * optimization; the DAG's sort+take downstream produces the final + * order/limit either way. */ + { + ray_group_emit_filter_t ef = ray_group_emit_filter_get(); + if (ray_agg_engine_v2 && group_limit == 0 && ef.enabled + && ext->n_keys == 1 + && (ef.agg_op == 0 || ef.agg_op == OP_COUNT || ef.agg_op == OP_SUM + || ef.agg_op == OP_MIN || ef.agg_op == OP_MAX) + && agg_v2_can_handle(g, op, tbl)) { + ray_op_t* k0 = op_node(g, ext->keys[0]); + ray_op_ext_t* k0e = k0 ? find_ext(g, k0->id) : NULL; + ray_t* k0c = (k0e && k0e->base.opcode == OP_SCAN) + ? ray_table_get_col(tbl, k0e->sym) : NULL; + /* Input-size gate: on RAW-table inputs (10M+ rows) consecutive + * rows repeat keys, so the serial dense scatter mostly hits + * cache and beats the radix pipeline (ClickBench q33/q34: + * 56ms serial vs 148ms via v2). The pathological case is the + * count-distinct SECOND phase, whose distinct-pairs + * intermediate (~1M rows) has no locality — every increment + * misses (q13: 88ms serial). Intermediates are bounded by + * their distinct count; raw fact tables are not. */ + if (k0c && k0c->type == RAY_SYM && + ray_table_nrows(tbl) <= (int64_t)(4u << 20) && + ray_sym_domain_count(ray_sym_vec_domain(k0c)) > (1 << 21)) { + /* Suppress the filter for the v2 run (v2 ignores it anyway; + * clearing keeps recursion/asserts honest), restore after. */ + ray_group_emit_filter_t saved = ray_group_emit_filter_get(); + ray_group_emit_filter_t off = {0}; + ray_group_emit_filter_set(off); + ray_t* r = exec_group_v2(g, op, tbl, 0); + ray_group_emit_filter_set(saved); + if (r && !RAY_IS_ERR(r)) + return group_emit_filter_trim(r, ext->n_keys, + ext->n_aggs, ef); + if (r) return r; + /* v2 declined at runtime — continue on the legacy ladder. */ + } + } + } /* v2 with EXPRESSION agg inputs: v2 admission requires plain-column * scans, so a group like {sum(a*b), stddev(c), cor(x,y)} — where ONE @@ -9178,9 +10423,9 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * v2's scan-input naming (agg_result_col_name) emits the SAME * `_e{a}_{op}` output names the legacy expression emit produced. * Any ineligibility falls through to the legacy path unchanged. */ - if (ray_agg_engine_v2 && group_limit == 0 + if (ray_agg_engine_v2 && group_limit >= 0 && !ray_group_emit_filter_get().enabled) { - ray_t* r = exec_group_v2_exprs(g, op, tbl); + ray_t* r = exec_group_v2_exprs(g, op, tbl, group_limit); if (r) return r; } @@ -9576,10 +10821,22 @@ static ray_t* exec_group_run(ray_graph_t* g, ray_op_t* op, ray_t* tbl, * reads the agg's int64 row slot directly. The non-COUNT paths * (sparse_i64 range-counting, the n_keys>1 macro fast path) still * gate on COUNT because they DON'T have the agg value available - * outside the row slot. */ + * outside the row slot. + * + * DIRECTION. The count-trimming machinery gated by use_emit_filter + * (da_count_emit_keep_min and the sparse/dense keep_min emits) is + * inherently LARGEST-first: it derives a minimum count to keep and drops + * every group below it. An `asc: take: N` shape wants the + * SMALLEST N, so those trims would discard exactly the rows the query + * asks for — they are disabled for it and the full group result flows to + * the downstream sort+take. The bounded-heap top-N path + * (use_topn_filter) is direction-symmetric (TOPN_BETTER) and serves asc + * shapes itself. A pure min_count_exclusive filter (no take) is + * direction-agnostic and stays enabled. */ bool use_emit_filter = emit_filter.enabled && emit_filter.agg_index < n_aggs && - ext->agg_ops[emit_filter.agg_index] == OP_COUNT; + ext->agg_ops[emit_filter.agg_index] == OP_COUNT && + (emit_filter.top_count_take <= 0 || emit_filter.desc); bool use_topn_filter = emit_filter.enabled && emit_filter.top_count_take > 0 && emit_filter.agg_index < n_aggs && @@ -10148,6 +11405,26 @@ da_path:; * carries per-agg arrays rather than a fixed-width bitmask. */ bool da_eligible = (n_scan > 0 && n_keys > 0 && n_keys <= 8 && n_aggs <= 64); + /* Selection-sparse gate (restored from the pre-#326 routing): with a + * bitmap rowsel and no flattened match_idx, both DA scans (min/max + * prescan and accumulate) visit every input row and test the bitmap + * per row — O(nrows) regardless of selectivity — while the HT/radix + * paths work off the surviving rows only. A selective filter makes + * the DA path arbitrarily worse than the input it replaces (ClickBench + * q07/q38/q40: 4-12ms -> 42-54ms), so route sparse selections away. */ + if (da_eligible && rowsel && !match_idx) { + ray_rowsel_t* sm = ray_rowsel_meta(rowsel); + if (sm && sm->total_pass * 4 < nrows) + da_eligible = false; + } + /* DA slot budget: dense per-worker state must stay well under the + * input size or the scatter loses to radix grouping on cache misses + * alone (ClickBench q28: a 2.7M-slot Referer array over 10M rows ran + * 10x slower than the HT path). n_scan/8 keeps the state O(input/8) + * while still allowing the >262144-slot shapes #326 opened up; the + * old fixed cap remains as the floor so small inputs keep DA. */ + uint64_t da_slot_budget = (uint64_t)(n_scan >> 3); + if (da_slot_budget < 262144) da_slot_budget = 262144; /* Binary aggregators (OP_PEARSON_CORR) are not wired into the * dense-array accumulator's per-worker da_accum_t struct — force * the HT path which has the row-layout offsets allocated. @@ -10231,12 +11508,12 @@ da_path:; if (key_scan_sym[k] < 0) continue; int64_t card = ray_grp_card_lookup(key_scan_sym[k]); if (card <= 1) continue; - if ((uint64_t)card > (uint64_t)n_scan / known_ub) { + if ((uint64_t)card > da_slot_budget / known_ub) { da_eligible = false; break; } known_ub *= (uint64_t)card; - if (known_ub > (uint64_t)n_scan) { da_eligible = false; break; } + if (known_ub > da_slot_budget) { da_eligible = false; break; } } } @@ -10270,7 +11547,7 @@ da_path:; .n_workers = mm_n, .match_idx = match_idx, .rowsel = rowsel, - .span_budget = n_scan, + .span_budget = (int64_t)da_slot_budget, .abort_flag = &mm_abort, }; if (mm_n > 1) { @@ -10295,8 +11572,8 @@ da_path:; da_key_range[k] = (int64_t)span; if (da_key_range[k] <= 0) { da_fits = false; break; } uint64_t range = (uint64_t)da_key_range[k]; - uint64_t slot_limit = (uint64_t)n_scan < INT32_MAX - ? (uint64_t)n_scan : INT32_MAX; + uint64_t slot_limit = da_slot_budget < INT32_MAX + ? da_slot_budget : INT32_MAX; if (total_slots > slot_limit / range) { da_fits = false; break; @@ -10989,7 +12266,30 @@ da_path:; uint8_t key_esz = ray_sym_elem_size(key_types[0], key_attrs[0]); - if (use_emit_filter && + /* Dense-cap vs survivor gate: the dyn-dense emit path callocs + * and slot-scans a key-range-sized array (1<<20 initial for + * 4-byte keys, up to 1<<24) — pure overhead when a WHERE + * leaves far fewer survivors than the array has slots + * (ClickBench q21: a 1M-slot scatter+scan for 44 rows). The + * comparison is cap-vs-survivors, NOT raw selectivity: a + * narrow key's 64K cap stays profitable at any filter (q07). + * Sparse-vs-cap selections continue to the radix/HT paths, + * which size their state from the surviving rows. */ + bool sp_dyn_sparse = false; + if (rowsel) { + ray_rowsel_t* sp_sm = ray_rowsel_meta(rowsel); + uint64_t sp_cap = key_esz == 1 ? 256u + : key_esz == 2 ? (1u << 16) + : (1u << 20); + /* Break-even multiplier: the dense side costs a sequential + * cap-sized calloc+scan (cheap per slot); the radix side + * costs per-survivor hashing plus fixed pipeline setup. + * Measured on ClickBench: 39K survivors amortize a 1M cap + * (q38, dense wins), 44 survivors do not (q21). */ + if (sp_sm && (uint64_t)sp_sm->total_pass * 32 < sp_cap) + sp_dyn_sparse = true; + } + if (use_emit_filter && !sp_dyn_sparse && (emit_filter.min_count_exclusive > 0 || emit_filter.top_count_take > 0) && n_scan <= UINT32_MAX) { @@ -11489,7 +12789,7 @@ ht_path:; ght_layout_t ght_layout; if (!ght_compute_layout(&ght_layout, n_keys, n_aggs, agg_vecs, agg_vecs2, ght_need, - ext->agg_ops, key_types)) { + ext->agg_ops, key_types, key_vecs)) { for (uint32_t a = 0; a < n_aggs; a++) { if (agg_owned[a] && agg_vecs[a]) ray_release(agg_vecs[a]); if (agg_owned2[a] && agg_vecs2[a]) ray_release(agg_vecs2[a]); } @@ -11505,7 +12805,19 @@ ht_path:; /* Parallel path: radix-partitioned group-by */ ray_pool_t* pool = ray_pool_get(); uint32_t n_total = pool ? ray_pool_total_workers(pool) : 1; - uint32_t radix_n_parts = group_radix_part_count(n_total, n_scan); + /* Size the radix fan-out by the rows that actually reach the partition + * HTs: with a pushed WHERE only the survivors are scattered, and a + * part count derived from the raw row count allocates and merges + * n_workers * n_parts hash tables of pure setup overhead (ClickBench + * q40: 8 x 4096 HTs for ~10K survivors — the merge dwarfed the query). + * Partition count only affects work distribution, never results. */ + int64_t radix_rows = n_scan; + if (g->selection) { + ray_rowsel_t* sel_sm = ray_rowsel_meta(g->selection); + if (sel_sm && sel_sm->nrows == nrows && sel_sm->total_pass < radix_rows) + radix_rows = sel_sm->total_pass > 0 ? sel_sm->total_pass : 1; + } + uint32_t radix_n_parts = group_radix_part_count(n_total, radix_rows); group_ht_t single_ht; group_ht_t top_ht; @@ -11521,6 +12833,99 @@ ht_path:; uint32_t* part_offsets = NULL; ray_t* group_out_hdr = NULL; uint32_t* group_out = NULL; + ray_t* topn_fuse_hdr = NULL; + ray_t* topn_fuse_cnt_hdr = NULL; + topn_fuse_t topn_fuse = {0}; + + /* ---- top-N ordering-agg resolution, hoisted above the parallel section. + * Phase 2 now runs each partition's top-k itself, while the partition is + * cache-hot (topn_fuse_partition), so the in-row offset of the ordering + * agg and the direction have to be known at DISPATCH time rather than + * derived after phase 2. The v2_emit compaction below consumes exactly + * these values, so the fused and standalone scans cannot drift apart. + * + * topn_order_ok == false means the shape is not servable by the int64 + * row-slot comparison (F64-output agg, SYM MIN/MAX, no value slot) — the + * whole compaction is skipped, exactly as the in-place `goto + * topn_compact_skip` did before. */ + uint16_t topn_order_off = 0; /* default: COUNT at row + 0 */ + uint8_t topn_desc_dir = 0; + bool topn_order_ok = false; + if (use_topn_filter) { + /* F64 agg outputs would have to compare by bitcast — for IEEE 754 + * that only preserves order for finite positive values, so they are + * excluded (COUNT is always I64, and SUM/MIN/MAX over an integer + * column keep an I64 slot; GHT_AF_F64 marks the SUM-over-F64 case). */ + uint16_t order_op = emit_filter.agg_op ? emit_filter.agg_op + : (uint16_t)OP_COUNT; + uint8_t ai = emit_filter.agg_index; /* < n_aggs, see use_topn_filter */ + bool order_is_f64 = (ght_layout.agg_flags[ai] & GHT_AF_F64) != 0; + int8_t agg_slot = ght_layout.agg_val_slot[ai]; + topn_order_ok = true; + if (order_op == OP_SUM) { + if (agg_slot < 0 || order_is_f64) topn_order_ok = false; + else topn_order_off = (uint16_t)(ght_layout.off_sum + + (uint16_t)agg_slot * 8u); + } else if (order_op == OP_MIN) { + if (agg_slot < 0 || order_is_f64 + || (ght_layout.agg_flags[ai] & GHT_AF_SYM)) topn_order_ok = false; + else topn_order_off = (uint16_t)(ght_layout.off_min + + (uint16_t)agg_slot * 8u); + } else if (order_op == OP_MAX) { + if (agg_slot < 0 || order_is_f64 + || (ght_layout.agg_flags[ai] & GHT_AF_SYM)) topn_order_ok = false; + else topn_order_off = (uint16_t)(ght_layout.off_max + + (uint16_t)agg_slot * 8u); + } + /* `.desc` is authoritative for every agg kind, COUNT included: every + * arming site in query.c sets it (1 for desc:, 0 for asc:). It used + * to be force-set to 1 for COUNT — that silently turned + * `asc: take: N` into the desc answer on this path while + * `-c 1` returned the smallest N (issue #408). */ + topn_desc_dir = emit_filter.desc ? 1 : 0; + } + + /* Admit the fused per-partition top-k: stash n_parts x k_take candidates, + * filled by phase 2 and consumed by the v2_emit merge below. + * + * The standalone pre-pass sizes its staging from the ACTUAL group counts + * and admits it only as a real reduction (<= an eighth of the groups). + * Fusion has to commit before any group exists, so the same test is made + * against the row count that bounds them: groups <= scattered rows, so + * n_parts * k_take <= radix_rows / 8 keeps the stash under 2 bytes per + * scattered row — against the >= 16 bytes per row phase 1's fat entries + * (or the v2 worker HTs) already hold, peak memory cannot move. Failing + * that test, or a single worker/partition, or allocation failure, leaves + * .items NULL and the unchanged post-phase-2 scan runs. */ + if (use_topn_filter && topn_order_ok && pool && n_total > 1 && + radix_n_parts > 1 && emit_filter.top_count_take > 0) { + uint64_t kt = (uint64_t)emit_filter.top_count_take; + uint64_t tot = (uint64_t)radix_n_parts * kt; + if (kt <= UINT32_MAX && tot <= (uint64_t)radix_rows / 8 && + tot <= UINT32_MAX && tot <= SIZE_MAX / sizeof(group_topn_item_t)) { + /* CALLOC, not alloc: item_cnt[] is written only by a phase-2 task, + * and ray_pool_dispatch_n drains its tickets WITHOUT running fn + * when the pool is cancelled — every entry must then read as a + * well-defined 0, not stack garbage that would send the merge off + * the end of items[]. */ + uint32_t* fcnt = (uint32_t*)scratch_calloc(&topn_fuse_cnt_hdr, + (size_t)radix_n_parts * sizeof(uint32_t)); + group_topn_item_t* fit = fcnt + ? (group_topn_item_t*)scratch_alloc(&topn_fuse_hdr, + (size_t)tot * sizeof(group_topn_item_t)) + : NULL; + if (fit) { + topn_fuse.items = fit; + topn_fuse.item_cnt = fcnt; + topn_fuse.cap = (uint32_t)kt; + topn_fuse.order_off = topn_order_off; + topn_fuse.desc_dir = topn_desc_dir; + } else if (fcnt) { + scratch_free(topn_fuse_cnt_hdr); + topn_fuse_cnt_hdr = NULL; + } + } + } /* Top-N-by-count (`select … by … desc:c take:N`) is served by the * parallel radix_v2 path below: phase1/phase2 build per-partition HTs @@ -11620,6 +13025,26 @@ ht_path:; } } } + /* Expected rows per (worker, partition) under a uniform hash — + * sized from the rows this dispatch will actually scatter. */ + uint64_t v2_rows_disp = (uint64_t)(sel_match ? sel_n : n_scan); + uint64_t v2_exp = v2_rows_disp + / ((uint64_t)n_total * radix_n_parts) + 1; + uint32_t v2_cap = 2; + while (v2_cap < v2_exp && v2_cap < 256) v2_cap <<= 1; + /* Slot-count ceiling for a worker HT that outgrows v2_cap: at the + * 50% rehash load factor a table receiving v2_exp rows needs at + * most 2*v2_exp slots, and v2_exp is already the + * per-(worker,partition) row budget computed above — no new + * tunable. This is a CEILING, not a target: group_ht_rehash + * approaches it one extra doubling at a time (see + * group_ht_t.grow_cap), because crossing a rung proves a lower + * bound on cardinality, not that growth will continue. Sizing the + * first allocation from it instead was measured and rejected: it + * cost q15 (density ~0.25) 5% and inflated worker-HT memory on + * every mid-cardinality shape. */ + uint32_t v2_grow = 2; + while (v2_grow < 2 * v2_exp && v2_grow < (1u << 24)) v2_grow <<= 1; radix_v2_phase1_ctx_t v2p1 = { .key_data = key_data, .key_types = key_types, @@ -11634,6 +13059,9 @@ ht_path:; .wpart_hts = wpart_hts, .rowsel = rowsel, .match_idx = sel_match, + .ht_init_cap = v2_cap, + .ht_init_block = v2_cap > 2 ? v2_cap / 2 : 1, + .ht_grow_cap = v2_grow, .oom = 0, }; /* By-value embed the layout, fixing its base pointers to v2p1's @@ -11658,6 +13086,7 @@ ht_path:; .n_workers = n_total, .n_parts = radix_n_parts, .key_data = key_data, + .fuse = topn_fuse, .oom = 0, }; ght_layout_copy(&v2p2.layout, &ght_layout); @@ -11729,6 +13158,8 @@ v2_done:; .bufs = radix_bufs, .rowsel = rowsel, .match_idx = match_idx, + .buf_prime = (uint32_t)((uint64_t)(n_scan > 0 ? n_scan : 1) + / ((uint64_t)n_total * radix_n_parts) * 5 / 4 + 8), }; ght_layout_copy(&p1ctx.layout, &ght_layout); /* Wide-key str-pool table (n_keys slots): carve once (never per row); @@ -11776,6 +13207,7 @@ v2_done:; .bufs = radix_bufs, .part_hts = part_hts, .key_data = key_data, + .fuse = topn_fuse, }; ght_layout_copy(&p2ctx.layout, &ght_layout); /* Wide-key str-pool table: phase2 copies each into its part_hts (whose @@ -11828,48 +13260,14 @@ v2_emit:; uint64_t total_pre = 0; for (uint32_t p = 0; p < radix_n_parts; p++) total_pre += part_hts[p].grp_count; - /* Resolve the in-row offset of the order-by agg's value. For - * COUNT it's the leading int64 at offset 0; for SUM/MIN/MAX - * it's the per-slot int64 in off_sum/off_min/off_max. F64 - * agg outputs (sum over an F64 column) compare by bitcast — - * for IEEE 754 the bit pattern preserves ordering for finite - * positive values; mixed-sign and NaN cases drop the heap - * back to a wider comparator. To stay correct we exclude - * F64-output aggs from this fast path (the COUNT count is - * always I64, and SUM/MIN/MAX over an integer column keep - * an I64 slot — agg_is_f64 marks the SUM-over-F64 case). */ - uint16_t order_op = emit_filter.agg_op - ? emit_filter.agg_op - : (uint16_t)OP_COUNT; - uint8_t agg_index_local = emit_filter.agg_index; - uint16_t order_off = 0; /* default: COUNT at row+0 */ - bool order_is_f64 = false; - if (agg_index_local < n_aggs && - (ght_layout.agg_flags[agg_index_local] & GHT_AF_F64)) - order_is_f64 = true; - int8_t agg_slot = ght_layout.agg_val_slot[agg_index_local]; - if (order_op == OP_SUM) { - if (agg_slot < 0 || order_is_f64) goto topn_compact_skip; - order_off = (uint16_t)(ght_layout.off_sum - + (uint16_t)agg_slot * 8u); - } else if (order_op == OP_MIN) { - if (agg_slot < 0 || order_is_f64) goto topn_compact_skip; - if (ght_layout.agg_flags[agg_index_local] & GHT_AF_SYM) - goto topn_compact_skip; - order_off = (uint16_t)(ght_layout.off_min - + (uint16_t)agg_slot * 8u); - } else if (order_op == OP_MAX) { - if (agg_slot < 0 || order_is_f64) goto topn_compact_skip; - if (ght_layout.agg_flags[agg_index_local] & GHT_AF_SYM) - goto topn_compact_skip; - order_off = (uint16_t)(ght_layout.off_max - + (uint16_t)agg_slot * 8u); - } - uint8_t desc_dir = emit_filter.desc ? 1 : 0; - /* COUNT defaults to desc when the filter struct's desc bit - * isn't set (old single-bit filter shape). Producer code in - * query.c sets it explicitly. */ - if (order_op == OP_COUNT && !emit_filter.desc) desc_dir = 1; + /* The in-row offset of the order-by agg's value and the sort + * direction are resolved ONCE, above the parallel section (search + * topn_order_off), because phase 2 needs them at dispatch time to + * run the fused per-partition scan. !topn_order_ok = a shape the + * int64 row-slot comparison cannot serve. */ + if (!topn_order_ok) goto topn_compact_skip; + uint16_t order_off = topn_order_off; + uint8_t desc_dir = topn_desc_dir; if (total_pre > (uint64_t)k_take && k_take > 0 && (uint64_t)k_take <= SIZE_MAX / sizeof(group_topn_item_t)) { /* The heap is sized by the requested result cardinality. @@ -11881,112 +13279,218 @@ v2_emit:; &heap_hdr, (size_t)k_take * sizeof(group_topn_item_t)); if (!heap) goto topn_compact_skip; int64_t hn = 0; - /* For top-N largest (desc=1): min-heap. Root is smallest; - * incoming v replaces root iff v > root. Heap invariant: - * parent ≤ child (so swap when parent > child). + + /* Each partition is reduced to its own <= k_take candidates + * and only those are merged here. Three sources, in + * preference order: * - * For top-N smallest (desc=0): max-heap. Root is largest; - * incoming v replaces root iff v < root. Heap invariant: - * parent ≥ child (so swap when parent < child). + * 1. FUSED — phase 2 already ran the scan at the end of the + * task that built each partition, while its rows were + * cache-hot. Nothing to dispatch: the candidates are in + * topn_fuse, at a uniform k_take stride per partition. + * 2. the standalone topn_scan_fn pre-pass, admitted when the + * staging array is a real REDUCTION — at most an eighth of + * the groups, i.e. under 2 bytes per group against the + * >= 24-byte group rows already resident, so peak memory + * cannot move. (Fusion could not make this test: it must + * commit before any group exists, so it makes the + * equivalent one against the row count — see topn_fuse.) + * 3. the serial scan below — one worker, one partition, + * allocation failure, or a staging array that would not + * reduce. * - * TOPN_NEEDS_SWAP(parent, child) := does the parent - * violate the invariant relative to child? */ - #define TOPN_NEEDS_SWAP(parent, child) \ - (desc_dir ? ((parent) > (child)) : ((parent) < (child))) - #define TOPN_SHOULD_REPLACE(new_v, root_v) \ - (desc_dir ? ((new_v) > (root_v)) : ((new_v) < (root_v))) - for (uint32_t p = 0; p < radix_n_parts; p++) { - group_ht_t* ph = &part_hts[p]; - uint16_t rs = ph->layout.row_stride; - uint32_t gc = ph->grp_count; - for (uint32_t gi = 0; gi < gc; gi++) { - const char* row = ph->rows + (size_t)gi * rs; - int64_t v = *(const int64_t*)(const void*) - (row + order_off); - if (hn < k_take) { - int64_t j = hn++; - heap[j] = (group_topn_item_t){v, p, gi}; - /* Sift up: bubble new entry toward root while - * parent violates invariant. */ - while (j > 0) { - int64_t pr = (j - 1) >> 1; - if (!TOPN_NEEDS_SWAP(heap[pr].value, - heap[j].value)) break; - group_topn_item_t tmp = heap[pr]; - heap[pr] = heap[j]; heap[j] = tmp; - j = pr; + * All three feed the same heap in the same (partition asc, + * gid asc) order, so they retain the same set. */ + ray_t* stage_hdr = NULL; + ray_t* item_hdr = NULL; + const uint32_t* item_off = NULL; /* NULL => uniform stride */ + const uint32_t* item_cnt = NULL; + const group_topn_item_t* items = NULL; + bool scanned = false; + if (topn_fuse.items) { + /* The cancel bail that guards this stash is the + * CHECK_CANCEL_GOTO right after the phase-2 dispatch that + * fills it; a drained dispatch never reaches here. Even + * so item_cnt is calloc'd and each task zeroes its own + * entry before any early `continue`, so a partition with + * no task contributes nothing rather than garbage. */ + items = topn_fuse.items; + item_cnt = topn_fuse.item_cnt; + scanned = true; + } else if (pool && n_total > 1 && radix_n_parts > 1 && + (size_t)radix_n_parts + 1 <= SIZE_MAX / sizeof(uint32_t) / 2) { + /* Per-partition capacity: at most k_take (a partition can + * contribute no more than that to the global top-k), and + * never more groups than it holds. 64-bit because k_take + * is an int64 LIMIT: match_group_desc_count_take caps it + * at 1024 but the count-distinct rewrite (query.c) does + * not, so the width is a guard, not a live shape. */ + const uint64_t kcap = (uint64_t)k_take; + uint64_t tot = 0; + for (uint32_t p = 0; p < radix_n_parts; p++) { + uint64_t gc = part_hts[p].grp_count; + tot += gc < kcap ? gc : kcap; + } + if (tot > 0 && tot <= total_pre / 8 && tot <= UINT32_MAX && + tot <= SIZE_MAX / sizeof(group_topn_item_t)) { + /* CALLOC, not alloc: item_cnt[] is written only by + * topn_scan_fn, and a CANCELLED dispatch drains its + * tickets WITHOUT running fn — every entry must then + * read as a well-defined 0, not stack garbage that + * would send the merge off the end of items[]. */ + uint32_t* off_w = (uint32_t*)scratch_calloc(&stage_hdr, + ((size_t)radix_n_parts * 2 + 1) * sizeof(uint32_t)); + group_topn_item_t* items_w = off_w + ? (group_topn_item_t*)scratch_alloc(&item_hdr, + (size_t)tot * sizeof(group_topn_item_t)) + : NULL; + if (items_w) { + uint32_t* cnt_w = off_w + radix_n_parts + 1; + uint32_t run = 0; + for (uint32_t p = 0; p < radix_n_parts; p++) { + uint64_t gc = part_hts[p].grp_count; + off_w[p] = run; + /* Same cap as the sizing pass above; the sum + * is `tot`, already checked <= UINT32_MAX. */ + run += (uint32_t)(gc < kcap ? gc : kcap); } - } else if (TOPN_SHOULD_REPLACE(v, heap[0].value)) { - heap[0] = (group_topn_item_t){v, p, gi}; - int64_t j = 0; - /* Sift down: find the child that should be - * promoted (the one most violating the - * invariant) and swap. */ - for (;;) { - int64_t l = j * 2 + 1, r = l + 1, m = j; - if (l < hn && TOPN_NEEDS_SWAP(heap[m].value, - heap[l].value)) m = l; - if (r < hn && TOPN_NEEDS_SWAP(heap[m].value, - heap[r].value)) m = r; - if (m == j) break; - group_topn_item_t tmp = heap[m]; - heap[m] = heap[j]; heap[j] = tmp; - j = m; + off_w[radix_n_parts] = run; + items = items_w; + item_off = off_w; + item_cnt = cnt_w; + topn_scan_ctx_t sctx = { + .part_hts = part_hts, .items = items_w, + .item_off = off_w, .item_cnt = cnt_w, + .order_off = order_off, .desc_dir = desc_dir, + }; + ray_pool_dispatch_n(pool, topn_scan_fn, &sctx, + radix_n_parts); + /* Same bail as every other dispatch in this + * driver (CHECK_CANCEL_GOTO), spelled out because + * this block owns scratch the cleanup label does + * not know about. A cancelled dispatch runs no + * tasks at all, so the survivors below would be + * empty/partial — bail rather than emit a wrong + * top-N. */ + if (pool_cancelled(pool)) { + scratch_free(item_hdr); + scratch_free(stage_hdr); + scratch_free(heap_hdr); + result = ray_error("cancel", NULL); + goto cleanup; } + scanned = true; } } } - #undef TOPN_NEEDS_SWAP - #undef TOPN_SHOULD_REPLACE + + /* Merge the per-partition survivors: partitions ascending, + * gid ascending within one — the serial scan's push order, + * which is what makes the retained set identical (condition + * (2) of the topn_heap_push proof). `item_off` is the + * standalone pre-pass's packed layout; the fused stash uses a + * uniform k_take stride instead. */ + if (scanned) + for (uint32_t p = 0; p < radix_n_parts; p++) { + const group_topn_item_t* h = items + + (item_off ? (size_t)item_off[p] + : (size_t)p * topn_fuse.cap); + uint32_t n = item_cnt[p]; + for (uint32_t i = 0; i < n; i++) + hn = topn_heap_push(heap, hn, k_take, desc_dir, + h[i].value, h[i].part, + h[i].gid); + } + scratch_free(item_hdr); + scratch_free(stage_hdr); + + if (!scanned) + for (uint32_t p = 0; p < radix_n_parts; p++) { + group_ht_t* ph = &part_hts[p]; + uint16_t rs = ph->layout.row_stride; + uint32_t gc = ph->grp_count; + for (uint32_t gi = 0; gi < gc; gi++) { + int64_t v = *(const int64_t*)(const void*) + (ph->rows + (size_t)gi * rs + order_off); + hn = topn_heap_push(heap, hn, k_take, desc_dir, + v, p, gi); + } + } + if (hn > 0) { - uint64_t all_groups = 0; - for (uint32_t p = 0; p < radix_n_parts; p++) - all_groups += part_hts[p].grp_count; - ray_t* keep_hdr = NULL; - ray_t* off_hdr = NULL; - uint8_t* keep = all_groups <= SIZE_MAX - ? (uint8_t*)scratch_calloc(&keep_hdr, - (size_t)all_groups) - : NULL; - uint32_t* keep_off = - (size_t)radix_n_parts + 1 <= SIZE_MAX / sizeof(uint32_t) - ? (uint32_t*)scratch_alloc(&off_hdr, - ((size_t)radix_n_parts + 1) * sizeof(uint32_t)) - : NULL; - if (keep && keep_off) { - keep_off[0] = 0; - for (uint32_t p = 0; p < radix_n_parts; p++) - keep_off[p + 1] = keep_off[p] - + part_hts[p].grp_count; - for (int64_t i = 0; i < hn; i++) - keep[keep_off[heap[i].part] + heap[i].gid] = 1; - - /* In-place compact each partition in original gid - * order, preserving deterministic tie behaviour. */ - bool rebuilt_slots = false; + /* Compact straight from the <= k_take survivors: ordering + * them by (partition, gid) makes each partition's keepers + * ascending, so the in-place move is the same one the old + * keep-bitmap walk made — but O(k_take + n_parts) instead + * of O(total groups), and with no bitmap to allocate. */ + qsort(heap, (size_t)hn, sizeof(*heap), topn_part_gid_cmp); + ray_t* dirty_hdr = NULL; + uint8_t* dirty = (uint8_t*)scratch_calloc(&dirty_hdr, + (size_t)radix_n_parts); + if (dirty) { + bool any_dirty = false; + int64_t i = 0; for (uint32_t p = 0; p < radix_n_parts; p++) { group_ht_t* ph = &part_hts[p]; uint16_t rs = ph->layout.row_stride; - uint32_t old_n = ph->grp_count; uint32_t kn = 0; - for (uint32_t gi = 0; gi < old_n; gi++) { - if (!keep[keep_off[p] + gi]) continue; + for (; i < hn && heap[i].part == p; i++) { + uint32_t gi = heap[i].gid; if (gi != kn) memmove(ph->rows + (size_t)kn * rs, ph->rows + (size_t)gi * rs, rs); kn++; } if (kn == ph->grp_count) continue; - rebuilt_slots = true; ph->grp_count = kn; + /* Resize the slot array to the survivors before it + * is rebuilt. A partition that held millions of + * groups keeps at most k_take of them, yet the + * rebuild's clear is O(ht_cap) — across all + * partitions of a 100M-row group-by that is a + * gigabyte of memset for a handful of live rows. + * The ALLOCATION is untouched (group_ht_free goes + * through the block header) and only shrinks, so + * every slot the table now uses is still inside it. + * Safe because nothing inserts into these tables + * again: phase 3 reads rows, and the holistic + * re-probe only probes. 2*kn keeps the same 50% + * load factor group_ht_rehash targets. */ + uint64_t need = (uint64_t)kn * 2; + uint32_t want = 2; + while ((uint64_t)want < need && want < ph->ht_cap) + want <<= 1; + if (want < ph->ht_cap) ph->ht_cap = want; + dirty[p] = 1; + any_dirty = true; } - if (rebuilt_slots) { - for (uint32_t p = 0; p < radix_n_parts; p++) - group_ht_rebuild_slots(&part_hts[p], key_types); + if (any_dirty) { + if (pool && n_total > 1 && radix_n_parts > 1) { + topn_rebuild_ctx_t rctx = { + .part_hts = part_hts, + .key_types = key_types, .dirty = dirty, + }; + ray_pool_dispatch_n(pool, topn_rebuild_fn, + &rctx, radix_n_parts); + /* A cancelled dispatch skips the rebuild and + * leaves slots pointing at pre-compaction + * gids; bail like every other dispatch here + * rather than carry stale tables forward. */ + if (pool_cancelled(pool)) { + scratch_free(dirty_hdr); + scratch_free(heap_hdr); + result = ray_error("cancel", NULL); + goto cleanup; + } + } else { + for (uint32_t p = 0; p < radix_n_parts; p++) + if (dirty[p]) + group_ht_rebuild_slots(&part_hts[p], + key_types); + } } } - scratch_free(off_hdr); - scratch_free(keep_hdr); + scratch_free(dirty_hdr); } scratch_free(heap_hdr); } @@ -12018,6 +13522,59 @@ v2_emit:; result = ray_error("limit", "group row index is too large"); goto cleanup; } + /* Sparse path: with few groups relative to rows, sort the + * (first_row, flat) pairs instead of building + scanning a + * dense uint32[nrows] map — the dense pass costs O(nrows) in + * allocation, memset and scan regardless of group count + * (ClickBench q40: a 40MB map and full 10M-row walk to order + * 110 groups). Sorting by earliest source row assigns exactly + * the ids the row-domain scan would. Duplicate first rows are + * rejected the same way the dense map's collision check did. */ + if ((uint64_t)total_grps <= row_count / 16) { + ray_t* pair_hdr = NULL; + group_order_pair_t* pairs = (group_order_pair_t*)scratch_alloc( + &pair_hdr, (size_t)total_grps * sizeof(group_order_pair_t)); + if (!pairs) { + result = ray_error("oom", NULL); + goto cleanup; + } + bool order_ok = true; + uint32_t np = 0; + for (uint32_t p = 0; p < radix_n_parts && order_ok; p++) { + group_ht_t* ph = &part_hts[p]; + for (uint32_t gi = 0; gi < ph->grp_count; gi++) { + uint32_t flat = part_offsets[p] + gi; + const char* group_row = ph->rows + + (size_t)gi * ph->layout.row_stride; + int64_t first; + memcpy(&first, + group_row + ph->layout.off_group_first, 8); + if (first < 0 || first >= nrows) { + order_ok = false; + break; + } + pairs[np].first = first; + pairs[np].flat = flat; + np++; + } + } + if (order_ok) { + qsort(pairs, np, sizeof(group_order_pair_t), + group_order_pair_cmp); + for (uint32_t i = 0; i < np; i++) { + if (i > 0 && pairs[i].first == pairs[i - 1].first) { + order_ok = false; + break; + } + group_out[pairs[i].flat] = i; + } + } + scratch_free(pair_hdr); + if (!order_ok || np != total_grps) { + result = ray_error("group", "failed to order all groups"); + goto cleanup; + } + } else { ray_t* first_hdr = NULL; uint32_t* first_to_flat = (uint32_t*)scratch_alloc( &first_hdr, (size_t)row_count * sizeof(uint32_t)); @@ -12056,6 +13613,7 @@ v2_emit:; result = ray_error("group", "failed to order all groups"); goto cleanup; } + } } /* Build result directly from partition HTs */ @@ -12521,13 +14079,16 @@ sequential_fallback:; const char* src_base = is_wide ? (const char*)key_data[k] : NULL; bool inline_str_k = (ly->key_flags[k] & GHT_KEYF_INLINE_STR) != 0; - /* Key k's null bit lives in null-mask word (k>>6), bit (k&63). */ - size_t null_woff = (size_t)ly->key_off[n_keys] + (size_t)(k >> 6) * 8; + /* Key k's null bit lives in null-mask word (k>>6), bit (k&63). An + * elided mask (null_words == 0) resolves to the shared zero word. */ + size_t null_woff = ly->null_words + ? (size_t)ly->key_off[n_keys] + (size_t)(k >> 6) * 8 : 0; int64_t null_kbit = (int64_t)((uint64_t)1 << (k & 63)); for (uint32_t gi = 0; gi < grp_count; gi++) { const char* row = final_ht->rows + (size_t)gi * ly->row_stride; const char* rk = row + 8; - int64_t null_word = *(const int64_t*)(rk + null_woff); + int64_t null_word = ly->null_words + ? *(const int64_t*)(rk + null_woff) : 0; if (null_word & null_kbit) { ray_vec_set_null(new_col, (int64_t)gi, true); /* Fill the correct-width sentinel. */ @@ -12644,6 +14205,23 @@ sequential_fallback:; int64_t* nullw = ek_buf + n_keys; /* null-mask words at key_off[nk] */ uint32_t null_words = ly->null_words; for (uint32_t w = 0; w < null_words; w++) nullw[w] = 0; + if (ly->packed_key) { + /* Packed tuple (ly->packed_key): every key is a plain integer lane, so + * the wide/F64 arms of the generic loop below are dead and the whole + * key region — values plus null-mask words — avalanches in ONE + * ght_hash_lanes call instead of nk hashes plus nk-1 combines. MUST + * stay in lockstep with hash_keys_inline (rehash/merge/lookup). */ + for (uint32_t k = 0; k < n_keys; k++) { + if (__builtin_expect(reprobe_nullable_s && ray_key_may_be_null(key_vecs[k]) + && ray_vec_is_null(key_vecs[k], row), 0)) { + nullw[k >> 6] |= (int64_t)((uint64_t)1 << (k & 63)); + ek_buf[k] = 0; + } else { + ek_buf[k] = read_col_i64(key_data[k], row, key_types[k], key_attrs[k]); + } + } + h = ght_hash_lanes(ek_buf, (uint32_t)n_keys + null_words); + } else { for (uint32_t k = 0; k < n_keys; k++) { int8_t t = key_types[k]; uint64_t kh; @@ -12657,10 +14235,12 @@ sequential_fallback:; ek_buf[k] = row; kh = wide_key_hash_at(ly, k, key_data, reprobe_pool, row); } else if (t == RAY_F64) { - int64_t kv; - memcpy(&kv, &((double*)key_data[k])[row], 8); + /* -0.0 -> +0.0 before BOTH the store and the hash + * — must match the builder that filled final_ht. */ + int64_t kv = group_key_f64_bits(key_data[k], row); + double dv; memcpy(&dv, &kv, 8); ek_buf[k] = kv; - kh = ray_hash_f64(((double*)key_data[k])[row]); + kh = ray_hash_f64(dv); } else { int64_t kv = read_col_i64(key_data[k], row, t, key_attrs[k]); ek_buf[k] = kv; @@ -12669,6 +14249,7 @@ sequential_fallback:; h = (k == 0) ? kh : ray_hash_combine(h, kh); } h = ght_hash_null_words(h, nullw, null_words); + } lookup_keys = ek_buf; } uint32_t gid = group_ht_lookup_gid(final_ht, h, lookup_keys, key_types); @@ -12808,7 +14389,8 @@ sequential_fallback:; int64_t cnt = *(const int64_t*)(const void*)row; /* nn = per-slot non-null count (nullable layout) or the group row * count (null-free — byte-identical to before). */ - int64_t nn = ly->off_nn + /* s < 0 for a valueless agg (OP_COUNT) — see radix_phase3_fn. */ + int64_t nn = (ly->off_nn && s >= 0) ? ((const int64_t*)(const void*)(row + ly->off_nn))[s] : cnt; if (out_type == RAY_F64) { double v; @@ -13008,6 +14590,8 @@ sequential_fallback:; } scratch_free(part_offsets_hdr); scratch_free(group_out_hdr); + scratch_free(topn_fuse_hdr); + scratch_free(topn_fuse_cnt_hdr); /* Master layout owns any spill block; every by-value copy borrowed it. * cleanup: is reached only after ght_layout is initialised (all gotos to * it are below the ght_compute_layout call). NULL-safe / no-op inline. */ @@ -14031,6 +15615,8 @@ bool pivot_ingest_run(pivot_ingest_t* out, .n_parts = n_parts, .bufs = radix_bufs, .match_idx = NULL, + .buf_prime = (uint32_t)((uint64_t)(n_scan > 0 ? n_scan : 1) + / ((uint64_t)n_total * n_parts) * 5 / 4 + 8), }; ght_layout_copy(&p1ctx.layout, ly); /* Wide-key str-pool table (n_keys slots): carved once, freed post-dispatch. */ diff --git a/src/ops/internal.h b/src/ops/internal.h index 9699add5..8704c7fe 100644 --- a/src/ops/internal.h +++ b/src/ops/internal.h @@ -1285,6 +1285,15 @@ typedef struct { uint8_t agg_flags_any; /* OR of all agg_flags[a] — scalar shape guard */ uint8_t any_wide_key; /* replaces wide_key_mask != 0 */ uint8_t any_inline_str; /* replaces key_inline_str != 0 */ + /* Set when the whole key tuple is plain fixed-width integer lanes: no + * wide/inline-STR key and no floating-point key. Such a key region is + * exactly (n_keys + null_words) 8-byte lanes whose STORED BITS are the + * identity, so hash and compare may treat it as one packed value. F64 + * keys are excluded: read_col_i64 (which the packed stagers use) does not + * decode F64 at all, and keeping them out also keeps the packed lanes free + * of the -0.0 canonicalisation the F64 builders apply (#407, + * group_key_f64_bits). Pinned by group_extra/ght_packed_key_excludes_f64. */ + uint8_t packed_key; uint8_t any_agg_null; /* OR of GHT_AF2_NULLABLE over aggs — hoisted hot-loop gate */ /* ── base pointers: aim at the *_in inline arrays (≤8) or the spill block ── */ int8_t* agg_val_slot; /* [n_aggs] accum slot per agg, -1 = none */ @@ -1367,6 +1376,24 @@ typedef struct { ray_t* _h_slots; ray_t* _h_rows; uint8_t oom; /* set by group_probe_entry on grow failure */ + /* Optional growth target in SLOTS (0 = plain doubling). Consumed ONLY by + * group_ht_rehash — never by group_ht_grow, which keeps the row array, + * where nearly all the bytes live, at O(groups). + * + * What the trigger actually proves: a rehash means the table just crossed + * ht_cap/2 groups. That is a lower bound on its cardinality, NOT evidence + * that it will keep growing — so the caller's target (derived from how + * many rows the table can still receive) is applied at most ONE extra + * doubling per rehash rather than in full. A table that stalls right + * after a jump therefore over-allocates its slot array by 2x and nothing + * else, while a genuinely near-unique table still reaches its final size + * in far fewer rungs — and every rung skipped is one whole re-hash and + * re-insert of every live group avoided. + * + * Idempotent (the jump is a max()), so once ht_cap reaches grow_cap the + * field stops having any effect. Placed here to land in the existing tail + * padding — group_ht_t's size is unchanged. */ + uint32_t grow_cap; } group_ht_t; /* Row-level accessors for group HT rows */ @@ -1402,7 +1429,8 @@ bool ght_compute_layout(ght_layout_t* out, uint32_t n_keys, uint32_t n_aggs, ray_t** agg_vecs, ray_t** agg_vecs2, uint8_t need_flags, const uint16_t* agg_ops, - const int8_t* key_types); + const int8_t* key_types, + ray_t* const* key_vecs); /* By-value copy that fixes the base pointers. Dispatches on STORAGE, not * ownership (src->spill_hdr == NULL is true both for genuine inline layouts * AND for borrowers, so it cannot be the test): when src->agg_val_slot == @@ -1437,7 +1465,12 @@ typedef struct { * For COUNT/SUM/MAX the natural ordering is largest-first; for * MIN it's smallest-first. Both directions are supported per * agg kind so `desc: min_value take: N` (the N groups with the - * largest min) is also expressible. */ + * largest min) is also expressible. + * + * CONTRACT (#408): every arming site MUST set this explicitly. + * A zero default is NOT coerced to largest-first for COUNT any + * more — consumers take .desc at face value, and the desc-only + * keep-min trims are gated off entirely when it is 0. */ uint8_t desc; } ray_group_emit_filter_t; ray_group_emit_filter_t ray_group_emit_filter_get(void); diff --git a/src/ops/pivot.c b/src/ops/pivot.c index acb045dc..8587d9d3 100644 --- a/src/ops/pivot.c +++ b/src/ops/pivot.c @@ -1026,8 +1026,12 @@ ray_t* exec_pivot(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { * point; every exit below (early return AND pivot_cleanup) must call * it exactly once so a spilled layout's heap block is never leaked. */ ght_layout_t ly; + /* key_vecs = NULL: pivot must NOT get the null-mask elision. Its ingest + * reads the key region's null word RAW (`keys[n_keys + pvt_null_word]` + * below) to detect a null pivot key, so the layout needs null_words >= 1. + * Passing NULL is what keeps it. */ if (!ght_compute_layout(&ly, n_keys, 1, agg_vecs, NULL, - need_flags, agg_ops, key_types)) { + need_flags, agg_ops, key_types, NULL)) { scratch_free(key_hdr); return ray_error("limit", "pivot: key stride budget exceeded"); } @@ -1400,7 +1404,13 @@ ray_t* exec_pivot(ray_graph_t* g, ray_op_t* op, ray_t* tbl) { int64_t cnt = *(const int64_t*)(const void*)row; /* nn = per-slot non-null count (nullable value column) or the * group row count (null-free — byte-identical to before). */ - int64_t nn = ly.off_nn + /* s < 0 for an agg that reserves no value slot (holistic, or + * the now-valueless OP_COUNT): it owns no off_nn slot either, + * so never index with it — mirrors the guard in group.c's + * radix_phase3_fn / sequential emit. Safe today only because + * off_nn is 0 for every such layout; the guard makes it not + * depend on that coincidence. */ + int64_t nn = (ly.off_nn && s >= 0) ? ((const int64_t*)(const void*)(row + ly.off_nn))[s] : cnt; if (out_agg_type == RAY_F64) { diff --git a/src/ops/query.c b/src/ops/query.c index cab91802..e853c592 100644 --- a/src/ops/query.c +++ b/src/ops/query.c @@ -38,6 +38,7 @@ #include "ops/fused_group.h" #include "ops/fused_topk.h" #include "ops/hll.h" +#include "ops/cdfuse.h" /* ray_cd_fused — fused grouped count-distinct */ #include "ops/temporal.h" #include "core/profile.h" #include "table/sym.h" @@ -524,6 +525,44 @@ static bool quantile_literal_prob(ray_t* prob_expr, bool percentile, return true; } +/* True when any GROUP BY key names a BOOL column of `tbl`. A BOOL-keyed + * group is emitted in key order (false before true) and query.c reorders the + * finished result to first-occurrence AFTERWARDS, so a take must not be pushed + * into the DAG ahead of that fix-up. */ +static bool group_keys_have_bool(ray_t* by_expr, ray_t* tbl) { + if (!by_expr || !tbl || RAY_IS_ERR(tbl) || tbl->type != RAY_TABLE) return true; + if (by_expr->type == -RAY_SYM && !(by_expr->attrs & ATTR_QUOTED)) { + ray_t* c = ray_table_get_col(tbl, by_expr->i64); + return !c || c->type == RAY_BOOL; + } + if (ray_is_vec(by_expr) && by_expr->type == RAY_SYM) { + const int64_t* ids = (const int64_t*)ray_data(by_expr); + for (int64_t i = 0; i < ray_len(by_expr); i++) { + ray_t* c = ray_table_get_col(tbl, ids[i]); + if (!c || c->type == RAY_BOOL) return true; + } + return false; + } + return true; /* computed / unresolved key shapes: stay conservative */ +} + +/* True when `tbl` carries PARTED or MAPCOMMON columns (a parted store). The + * parted GROUP dispatch (exec_group_parted) treats a positive group_limit as + * "stop after group_limit partitions", which assumes every partition yields a + * group — an EMPTY partition breaks that and under-fills the answer. Rather + * than push a hint that path would mis-apply, grouped takes over parted inputs + * keep the post-execution take. */ +static bool table_is_parted(ray_t* tbl) { + if (!tbl || RAY_IS_ERR(tbl) || tbl->type != RAY_TABLE) return true; + int64_t nc = ray_table_ncols(tbl); + for (int64_t c = 0; c < nc; c++) { + ray_t* col = ray_table_get_col_idx(tbl, c); + if (col && (RAY_IS_PARTED(col->type) || col->type == RAY_MAPCOMMON)) + return true; + } + return false; +} + /* Apply sort (asc/desc) and take clauses to a materialized result table. * Used by eval-level paths that bypass the DAG (e.g., LIST/STR group keys). * Builds a temporary DAG for sorting (supports per-column direction flags) @@ -533,8 +572,21 @@ static bool quantile_literal_prob(ray_t* prob_expr, bool percentile, * name), an atom take with K << nrows, and the result is a flat table * with no LIST columns, dispatch to ray_topk_table — bounded-heap * selection in O(n log K) instead of full sort + gather. */ +/* Decoded take: value the CALLER already evaluated (grouped DAG path: it + * evaluates take: once to decide the HEAD pushdown). Passing the DECODED form + * rather than the ray_t keeps it a stack value with no ownership to leak on the + * caller's error paths, and re-evaluating a side-effecting or expensive take: + * expression a second time is avoided either way. */ +typedef enum { TAKE_PRE_NONE = 0, TAKE_PRE_ATOM, TAKE_PRE_RANGE } take_pre_kind_t; +typedef struct { + take_pre_kind_t kind; + int64_t a, b; /* ATOM: a = count. RANGE: [a start, b amount]. */ +} take_pre_t; + +/* `take_pre` (may be NULL / TAKE_PRE_NONE) — see take_pre_t. */ static ray_t* apply_sort_take(ray_t* result, ray_t** dict_elems, int64_t dict_n, - int64_t asc_id, int64_t desc_id, int64_t take_id) { + int64_t asc_id, int64_t desc_id, int64_t take_id, + const take_pre_t* take_pre) { if (!result || RAY_IS_ERR(result)) return result; /* Check for sort/take clauses */ @@ -548,7 +600,22 @@ static ray_t* apply_sort_take(ray_t* result, ray_t** dict_elems, int64_t dict_n, if (!has_sort && !take_val_expr) return result; if (!has_sort && take_val_expr) { - ray_t* tv = ray_eval(take_val_expr); + /* Re-materialize the caller's already-evaluated value instead of + * running take_val_expr again; the rest of this branch is unchanged + * and owns tv exactly as before. */ + ray_t* tv; + if (take_pre && take_pre->kind == TAKE_PRE_ATOM) { + tv = ray_i64(take_pre->a); + } else if (take_pre && take_pre->kind == TAKE_PRE_RANGE) { + tv = ray_vec_new(RAY_I64, 2); + if (tv && !RAY_IS_ERR(tv)) { + ((int64_t*)ray_data(tv))[0] = take_pre->a; + ((int64_t*)ray_data(tv))[1] = take_pre->b; + tv->len = 2; + } + } else { + tv = ray_eval(take_val_expr); + } if (!tv || RAY_IS_ERR(tv)) { ray_release(result); return tv ? tv : ray_error("domain", "select: failed to evaluate `take:`"); @@ -2006,6 +2073,7 @@ static int expr_contains_call_named(ray_t* expr, const char* name, size_t name_l } static ray_t* query_materialize_parted_col(ray_t* col); +static bool table_has_parted_columns(ray_t* tbl); /* True when a projection's TOP-LEVEL call is a "whole-column verb": a * length-changing / reordering builtin (distinct, asc, desc, reverse) that @@ -2387,6 +2455,10 @@ static bool match_group_count_emit_filter(ray_t* from_expr, ray_t* where_expr, out->enabled = 1; out->agg_index = agg_index; out->min_count_exclusive = threshold; + /* Keeps the LARGEST counts (count > threshold): consumers read + * .desc as the direction, so state it rather than relying on a + * zero default. */ + out->desc = 1; DICT_VIEW_CLOSE(iv); return true; } @@ -3658,6 +3730,63 @@ static ray_t* try_count_distinct_v2_rewrite( if (where_expr && !ray_fused_group_supported(where_expr, tbl)) return NULL; + /* === Fused single-pass kernel (spec Part B) === + * Single narrow key, plain column cd-inner, no WHERE, flat columns: + * ray_cd_fused replaces the two group-by passes below with one + * partitioned scatter + per-partition dedup. It declines (NULL) on + * any shape it does not handle — small inputs, unsupported types, + * nullable columns, allocation pressure — and NULL is never an + * error: the two-pass rewrite below then runs unchanged. + * + * The kernel calls ray_pool_dispatch unconditionally, and the pool is + * single-producer, so it must never run from inside an in-flight + * dispatch (i.e. from a worker thread). ray_pool_par_dispatch_ok + * carries exactly that check (n_workers > 0 && !ray_parallel_flag) + * plus the row-count floor. */ + if (n_K == 1 && !where_expr) { + int64_t nrows = ray_table_nrows(tbl); + if (ray_pool_par_dispatch_ok(ray_pool_get(), nrows, CDF_MIN_ROWS)) { + ray_t* fr = ray_cd_fused(K_cols[0], X_col, nrows); + if (fr) { + if (RAY_IS_ERR(fr)) return fr; + /* fr = (k, u, _first) I64 columns in first-seen key order. + * Emit {K: keys, c: counts}: the key column is rebuilt at + * the source column's own type/width (so a SYM key stays a + * SYM over the source domain and renders as text), and + * `_first` — an internal ordering artifact — is dropped by + * building a fresh 2-column table. */ + ray_t* fk = ray_table_get_col_idx(fr, 0); + ray_t* fu = ray_table_get_col_idx(fr, 1); + int64_t ng = fk ? fk->len : 0; + ray_t* kv = col_vec_new(K_cols[0], ng > 0 ? ng : 1); + if (!kv || RAY_IS_ERR(kv)) { + ray_release(fr); + return kv ? kv : ray_error("oom", NULL); + } + if (kv->type == RAY_SYM) + ray_sym_vec_adopt_domain(kv, sym_domain_rep(K_cols[0])); + kv->len = ng; + { + const int64_t* src = (const int64_t*)ray_data(fk); + void* dst = ray_data(kv); + for (int64_t i = 0; i < ng; i++) + write_col_i64(dst, i, src[i], kv->type, kv->attrs); + } + ray_t* out = ray_table_new(2); + if (out && !RAY_IS_ERR(out)) + out = ray_table_add_col(out, K_syms[0], kv); + if (out && !RAY_IS_ERR(out)) + out = ray_table_add_col(out, cd_c_sym, fu); + ray_release(kv); + ray_release(fr); + if (!out) return ray_error("oom", NULL); + if (RAY_IS_ERR(out)) return out; + return apply_sort_take(out, dict_elems, dict_n, + asc_id, desc_id, take_id, NULL); + } + } + } + /* === Inner pass: group by (K1, ..., Kn, X) on the source table === */ ray_graph_t* g_in = ray_graph_new(tbl); if (!g_in) return NULL; @@ -3735,6 +3864,11 @@ static ray_t* try_count_distinct_v2_rewrite( emit_f.agg_index = 0; emit_f.top_count_take = take_n; emit_f.min_count_exclusive = 0; + /* This rewrite fires only for `desc: c take: N` (see the + * desc_col_sym test above) — consumers read .desc as the direction, + * so set it explicitly instead of leaning on a zero default that + * used to be coerced to desc inside group.c. */ + emit_f.desc = 1; ray_group_emit_filter_set(emit_f); emit_set = 1; } @@ -3776,7 +3910,7 @@ static ray_t* try_count_distinct_v2_rewrite( * `desc: c` ordering is silently dropped — the result set is right * but its row order isn't. apply_sort_take is a no-op when the * clauses are absent. */ - return apply_sort_take(result, dict_elems, dict_n, asc_id, desc_id, take_id); + return apply_sort_take(result, dict_elems, dict_n, asc_id, desc_id, take_id, NULL); } /* Per-group count(distinct) using the existing OP_COUNT_DISTINCT kernel. @@ -4454,7 +4588,14 @@ static ray_t* eval_scalar_agg_outputs(ray_t** dict_elems, int64_t dict_n, fn_obj ? ray_type_name(fn_obj->type) : "null"); } - ray_t* src = eval_expr_per_row(agg_elems[1], tbl, nrows); + /* A whole-column verb (distinct/asc/desc/reverse) at the head of the + * aggregate's argument consumes the entire column — per-row scatter + * would feed it one scalar cell at a time, making `count` report the + * row count instead of the distinct count (issue #405). Mirror the + * projection fallback's routing and evaluate it once. */ + ray_t* src = is_whole_column_projection(agg_elems[1]) + ? eval_expr_whole_column(agg_elems[1], tbl) + : eval_expr_per_row(agg_elems[1], tbl, nrows); if (!src || RAY_IS_ERR(src)) { ray_release(result); return src ? src : ray_error("domain", "select: failed to evaluate aggregation source"); @@ -6406,6 +6547,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { int n_compound = 0; int n_aggs_real = 0; /* DAG agg slots before hidden ones */ int synth_count_col = 0; /* 1 if we synthesized OP_COUNT for group boundaries */ + int has_list_agg = 0; /* 1 if any aggregate emits a LIST column (top/bot) */ if (where_expr && by_expr && !nearest_expr && can_defer_single_key_where(by_expr, where_expr, tbl)) { @@ -7715,7 +7857,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { scratch_free(sel_slots_hdr); DICT_VIEW_CLOSE(dv); return result; } result = apply_sort_take(result, dict_elems, dict_n, - asc_id, desc_id, take_id); + asc_id, desc_id, take_id, NULL); scratch_free(sel_slots_hdr); DICT_VIEW_CLOSE(dv); return result; } @@ -7908,7 +8050,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { scratch_free(sel_slots_hdr); DICT_VIEW_CLOSE(dv); return first_err; } res = apply_sort_take(res, dict_elems, dict_n, - asc_id, desc_id, take_id); + asc_id, desc_id, take_id, NULL); scratch_free(sel_slots_hdr); DICT_VIEW_CLOSE(dv); return res; } @@ -8348,7 +8490,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { if (eval_tbl != tbl) ray_release(eval_tbl); ray_release(tbl); result = apply_sort_take(result, dict_elems, dict_n, - asc_id, desc_id, take_id); + asc_id, desc_id, take_id, NULL); scratch_free(sel_slots_hdr); DICT_VIEW_CLOSE(dv); return result; } @@ -8683,6 +8825,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { } has_binary_agg = 1; } else if (op == OP_TOP_N || op == OP_BOT_N) { + has_list_agg = 1; if (ray_len(val_expr) < 3) { ray_graph_free(g); ray_release(tbl); scratch_free(sel_slots_hdr); DICT_VIEW_CLOSE(dv); return ray_error("arity", "select by: top/bot aggregation requires a K argument"); } ray_t* k_expr = agg_elems[2]; int64_t k_val; @@ -8750,6 +8893,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { agg_ins2[n_aggs] = NULL; agg_k[n_aggs] = 0; if (hop == OP_TOP_N || hop == OP_BOT_N) { + has_list_agg = 1; if (ray_len(hidden_agg_exprs[hi]) < 3) { for (int ci = 0; ci < n_compound; ci++) ray_release(compound_rw[ci]); @@ -9159,10 +9303,9 @@ ray_t* ray_select(ray_t** args, int64_t n) { * We also record a per-group null flag. The DAG GROUP path * stores null keys with value=0 and differentiates via a * null mask — if we hashed raw bits only, a null group would - * collide with non-null value 0 (for I64 / I32 / SYM / DATE - * / TIME etc.) or with +0.0 for F64 (ray_hash_f64 normalises - * -0.0 to +0.0, and F64's null bit pattern on this platform - * is the -0.0 pattern). The null flag keeps those groups + * collide with non-null value 0 (for I64 / I32 / SYM / DATE / + * TIME etc.), and for F64 with +0.0, since an all-zero key slot + * reads back as +0.0. The null flag keeps those groups * distinct. */ uint8_t gk_null_stack[256]; ray_t* gk_null_hdr = NULL; @@ -9184,18 +9327,27 @@ ray_t* ray_select(ray_t** args, int64_t n) { bool gk_has_nulls = (grp_key_col->attrs & RAY_ATTR_HAS_NULLS) != 0; if (gk_has_nulls) { for (int64_t gi = 0; gi < n_groups; gi++) { - if (kt == RAY_F64) - memcpy(&gk_vals[gi], &((double*)ray_data(grp_key_col))[gi], 8); - else + if (kt == RAY_F64) { + /* -0.0 -> +0.0, matching group.c's key-read + * normalisation (group_key_f64_bits, #407): + * gk_vals is compared BIT-WISE against the source + * column below while the probe hashes through + * ray_hash_f64 (which normalises), so both sides + * must carry canonical bits or a -0.0 row would + * never match its own +0.0 group. */ + double dv = clear_neg_zero(((double*)ray_data(grp_key_col))[gi]); + memcpy(&gk_vals[gi], &dv, 8); + } else gk_vals[gi] = read_col_i64(ray_data(grp_key_col), gi, kt, grp_key_col->attrs); if (ray_vec_is_null(grp_key_col, gi)) gk_null[gi] = 1; } } else { for (int64_t gi = 0; gi < n_groups; gi++) { - if (kt == RAY_F64) - memcpy(&gk_vals[gi], &((double*)ray_data(grp_key_col))[gi], 8); - else + if (kt == RAY_F64) { /* canonical -0.0, see above */ + double dv = clear_neg_zero(((double*)ray_data(grp_key_col))[gi]); + memcpy(&gk_vals[gi], &dv, 8); + } else gk_vals[gi] = read_col_i64(ray_data(grp_key_col), gi, kt, grp_key_col->attrs); } } @@ -9239,9 +9391,8 @@ ray_t* ray_select(ray_t** args, int64_t n) { * For F64 keys, hash via the float path; memcpy bit pattern * out of gk_vals to dodge strict-aliasing. Null groups * get a distinct hash so they don't collide with zero-valued - * groups (F64 null has the -0.0 bit pattern, which - * ray_hash_f64 normalises to +0.0; integer-flavoured - * nulls are stored as value=0). */ + * groups (null keys are stored as an all-zero key slot, + * which reads back as integer 0 / +0.0). */ for (int64_t gi = 0; gi < n_groups; gi++) { uint64_t h; if (gk_null[gi]) { @@ -9268,8 +9419,10 @@ ray_t* ray_select(ray_t** args, int64_t n) { for (int64_t r = 0; r < nrows_orig && found < n_groups; r++) { bool r_null = orig_nulls_flag && ray_vec_is_null(orig_key_col, r); int64_t ov; - if (kt == RAY_F64) memcpy(&ov, &((double*)ray_data(orig_key_col))[r], 8); - else ov = read_col_i64(ray_data(orig_key_col), r, kt, orig_key_col->attrs); + if (kt == RAY_F64) { /* canonical -0.0, see above */ + double dv = clear_neg_zero(((double*)ray_data(orig_key_col))[r]); + memcpy(&ov, &dv, 8); + } else ov = read_col_i64(ray_data(orig_key_col), r, kt, orig_key_col->attrs); uint64_t h; if (r_null) { h = ray_hash_i64((int64_t)0xDEADBEEFCAFEBABEULL); @@ -9399,7 +9552,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { if (filtered_tbl != tbl) ray_release(filtered_tbl); ray_release(tbl); result = apply_sort_take(result, dict_elems, dict_n, - asc_id, desc_id, take_id); + asc_id, desc_id, take_id, NULL); scratch_free(sel_slots_hdr); DICT_VIEW_CLOSE(dv); return result; } } else if (n_out > 0) { @@ -9664,7 +9817,7 @@ ray_t* ray_select(ray_t** args, int64_t n) { if (nearest_query_owned) ray_sys_free(nearest_query_owned); ray_graph_free(g); ray_release(tbl); result = apply_sort_take(result, dict_elems, dict_n, - asc_id, desc_id, take_id); + asc_id, desc_id, take_id, NULL); scratch_free(colops_hdr); scratch_free(sel_slots_hdr); DICT_VIEW_CLOSE(dv); return result; } else { @@ -9731,21 +9884,63 @@ ray_t* ray_select(ray_t** args, int64_t n) { scratch_free(sortk_hdr); } - /* Take: add to DAG only when no group-by and no nearest (rerank - * absorbs the take into its k parameter). */ + /* Take: added to the DAG when there is no nearest (rerank absorbs the take + * into its k parameter). + * + * Without a group-by the take is the DAG's HEAD/TAIL outright. WITH a + * group-by the result schema differs from the input, so the take is + * normally left to the post-execution apply_sort_take — EXCEPT for a + * positive integer atom take with no asc:/desc: clause: the group emits + * groups in stable first-seen order, so "first N rows of the group result" + * is exactly "first N groups". Adding a HEAD above the GROUP node lets + * exec.c's HEAD(GROUP) fusion forward N to exec_group as the group_limit + * HINT, which the v2 radix engine uses to emit only N groups instead of + * materializing all of them. The hint is advisory: HEAD still trims and + * apply_sort_take still runs at the end, so correctness never depends on + * it. NOT pushed with a group-by: negative (tail) and range takes, which + * both need the full group set; and any asc:/desc: shape, which reorders + * the groups first (that shape also owns the desc+take emit-filter + * machinery — has_sort keeps the two disjoint). Also NOT pushed when a + * deferred key WHERE runs AFTER the group (post_group_where_expr): that + * filter drops result rows, so taking the first N groups before it would + * under-fill the answer. Nor over a PARTED input (exec_group_parted + * mis-applies the hint on empty partitions — see table_is_parted), nor + * with a LIST-producing aggregate (top/bot): exec.c's HEAD/TAIL table trim + * now retains LIST cells, but the group's LIST output has no reason to pay + * a trim it can skip, and the take is a no-op there anyway. */ ray_t* take_range = NULL; - if (take_expr && !by_expr && !nearest_expr) { + take_pre_t take_pre = {0}; /* decoded take: value, reused by apply_sort_take */ + bool group_take_push = (take_expr && by_expr && !nearest_expr && !has_sort + && !post_group_where_expr && !has_list_agg + && !group_keys_have_bool(by_expr, tbl) + && !table_is_parted(tbl)); + if (take_expr && !nearest_expr && (!by_expr || group_take_push)) { ray_t* tv = ray_eval(take_expr); if (!tv || RAY_IS_ERR(tv)) { ray_graph_free(g); ray_release(tbl); scratch_free(sel_slots_hdr); DICT_VIEW_CLOSE(dv); return tv ? tv : ray_error("domain", "select: failed to evaluate `take:`"); } if (ray_is_atom(tv) && (tv->type == -RAY_I64 || tv->type == -RAY_I32)) { int64_t n_take = (tv->type == -RAY_I64) ? tv->i64 : tv->i32; + if (group_take_push) { + take_pre.kind = TAKE_PRE_ATOM; + take_pre.a = n_take; + } ray_release(tv); - if (n_take >= 0) + if (group_take_push) { + if (n_take > 0) root = ray_head(g, root, n_take); + /* n_take <= 0 with a group-by: leave it to apply_sort_take. */ + } else if (n_take >= 0) root = ray_head(g, root, n_take); else root = ray_tail(g, root, -n_take); } else if (ray_is_vec(tv) && (tv->type == RAY_I64 || tv->type == RAY_I32) && tv->len == 2) { - take_range = tv; /* apply after DAG execution */ + if (group_take_push) { + /* range: apply_sort_take slices it after execution */ + const int64_t* rv = (const int64_t*)ray_data(tv); + take_pre.kind = TAKE_PRE_RANGE; + if (tv->type == RAY_I64) { take_pre.a = rv[0]; take_pre.b = rv[1]; } + else { const int32_t* r32 = (const int32_t*)rv; + take_pre.a = r32[0]; take_pre.b = r32[1]; } + ray_release(tv); + } else take_range = tv; /* apply after DAG execution */ } else { int8_t tv_t = tv->type; /* capture BEFORE free */ ray_release(tv); @@ -10943,7 +11138,9 @@ ray_t* ray_select(ray_t** args, int64_t n) { * last so non-agg LIST columns are already in the result, * allowing sort clauses to reference non-agg output columns. */ if (by_expr && (has_sort || take_expr)) - result = apply_sort_take(result, dict_elems, dict_n, asc_id, desc_id, take_id); + result = apply_sort_take(result, dict_elems, dict_n, asc_id, desc_id, + take_id, + take_pre.kind ? &take_pre : NULL); if (by_sym_vec_owned) ray_release(by_sym_vec_owned); if (saved_selection) ray_release(saved_selection); @@ -11314,6 +11511,35 @@ ray_t* ray_update(ray_t** args, int64_t n) { } if (tbl->type != RAY_TABLE) { int8_t tbl_t = tbl->type; ray_release(tbl); return ray_error("type", "update: `from:` must be a table, got %s", ray_type_name(tbl_t)); } + /* A parted table's data columns carry the RAY_PARTED_BASE wrapper type + * (which `ray_type_name` prints as "?"), and its partition key is + * RAY_MAPCOMMON. The update machinery below reads the original column + * through `ray_vec_new(ct, ...)` / `ray_data(col)` / the per-group gather + * and type-check against the wrapper type, none of which understand the + * parted/segmented shape — so `(update {col: … from: partedT})` failed + * with `expression type I64 does not match ? column` (or the `by:` path + * with `group: argument must be a vector`). `select` solves this by + * materialising parted columns on demand; replicate it here by flattening + * the whole table once so every branch below sees wrapped-free vectors. */ + if (table_has_parted_columns(tbl)) { + ray_t* flat_tbl = ray_table_new(ray_table_ncols(tbl)); + if (!flat_tbl || RAY_IS_ERR(flat_tbl)) { ray_release(tbl); return flat_tbl ? flat_tbl : ray_error("oom", NULL); } + int64_t nc = ray_table_ncols(tbl); + for (int64_t c = 0; c < nc; c++) { + ray_t* col = ray_table_get_col_idx(tbl, c); + ray_t* flat_col = query_materialize_parted_col(col); + if (!flat_col || RAY_IS_ERR(flat_col)) { + ray_release(flat_tbl); ray_release(tbl); + return flat_col ? flat_col : ray_error("oom", NULL); + } + flat_tbl = ray_table_add_col(flat_tbl, ray_table_col_name(tbl, c), flat_col); + ray_release(flat_col); + if (!flat_tbl || RAY_IS_ERR(flat_tbl)) { ray_release(tbl); return flat_tbl ? flat_tbl : ray_error("oom", NULL); } + } + ray_release(tbl); + tbl = flat_tbl; + } + ray_t* where_expr = dict_get(dict, "where"); ray_t* by_expr = dict_get(dict, "by"); @@ -11368,37 +11594,55 @@ ray_t* ray_update(ray_t** args, int64_t n) { if (RAY_IS_ERR(groups)) { ray_release(tbl); DICT_VIEW_CLOSE(updv); return groups; } } - /* Start with a copy of the original table */ int64_t ncols = ray_table_ncols(tbl); - ray_t* result = ray_table_new((int32_t)ncols); - if (RAY_IS_ERR(result)) { ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return result; } - for (int64_t c = 0; c < ncols; c++) { - int64_t cn = ray_table_col_name(tbl, c); - ray_t* col = ray_table_get_col_idx(tbl, c); - ray_retain(col); - result = ray_table_add_col(result, cn, col); - ray_release(col); - if (RAY_IS_ERR(result)) { ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return result; } - } + int64_t ngroups = groups->len / 2; + ray_t** gdata = (ray_t**)ray_data(groups); + if (!gdata) { ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return ray_error("oom", NULL); } + + int64_t n_updates = 0; + for (int64_t d = 0; d + 1 < dict_n; d += 2) { + int64_t kid = dict_elems[d]->i64; + if (kid == from_id || kid == where_id || kid == by_id) continue; + n_updates++; + } + size_t upd_slots = (size_t)(n_updates ? n_updates : 1); + ray_t* upd_hdr = NULL; + int64_t* upd_names = (int64_t*)scratch_calloc(&upd_hdr, + upd_slots * sizeof(int64_t) + + upd_slots * sizeof(ray_t*) + + upd_slots * sizeof(uint8_t)); + if (!upd_names) { ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return ray_error("oom", NULL); } + ray_t** upd_cols = (ray_t**)(upd_names + upd_slots); + uint8_t* upd_used = (uint8_t*)(upd_cols + upd_slots); + + #define UPDATE_BY_CLEANUP_COLS() do { \ + for (int64_t _ui = 0; _ui < n_updates; _ui++) \ + if (upd_cols[_ui]) ray_release(upd_cols[_ui]); \ + scratch_free(upd_hdr); \ + } while (0) /* For each aggregate expression, compute per group and broadcast */ + int64_t upd_i = 0; for (int64_t d = 0; d + 1 < dict_n; d += 2) { int64_t kid = dict_elems[d]->i64; if (kid == from_id || kid == where_id || kid == by_id) continue; ray_t* agg_expr = dict_elems[d + 1]; - /* Evaluate the aggregate for each group and broadcast */ - ray_t* grp_items = (ray_t**)ray_data(groups) ? groups : NULL; - if (!grp_items) { ray_release(result); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return ray_error("oom", NULL); } - int64_t ngroups = groups->len / 2; - ray_t** gdata = (ray_t**)ray_data(groups); - /* We need to evaluate the aggregate per group. * Build the result column by evaluating the expression on each group's subset. */ - ray_t* out_col = ray_vec_new(RAY_I64, nrows2); /* will be resized to correct type */ - if (RAY_IS_ERR(out_col)) { ray_release(result); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return out_col; } - int8_t out_type = RAY_I64; + ray_t* target_col = ray_table_get_col(tbl, kid); + if (ngroups == 0 && target_col) out_type = target_col->type; + ray_t* out_col = ray_vec_new(out_type, nrows2); /* resized to expression type on first group */ + if (RAY_IS_ERR(out_col)) { UPDATE_BY_CLEANUP_COLS(); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return out_col; } + if (ngroups == 0) { + out_col->len = nrows2; + /* Match the per-group path's zero-fill so an unwritten buffer + * never carries allocator garbage (unreachable with rows today + * — a non-empty table always yields a group — but uniform). */ + memset(ray_data(out_col), 0, + (size_t)nrows2 * (size_t)ray_sym_elem_size(out_col->type, out_col->attrs)); + } int first_group = 1; for (int64_t gi = 0; gi < ngroups; gi++) { @@ -11407,7 +11651,7 @@ ray_t* ray_update(ray_t** args, int64_t n) { /* Build a sub-table for this group */ ray_t* sub_tbl = ray_table_new((int32_t)ncols); - if (RAY_IS_ERR(sub_tbl)) { ray_release(out_col); ray_release(result); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return sub_tbl; } + if (RAY_IS_ERR(sub_tbl)) { ray_release(out_col); UPDATE_BY_CLEANUP_COLS(); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return sub_tbl; } for (int64_t c = 0; c < ncols; c++) { int64_t cn = ray_table_col_name(tbl, c); ray_t* full_col = ray_table_get_col_idx(tbl, c); @@ -11421,7 +11665,7 @@ ray_t* ray_update(ray_t** args, int64_t n) { ray_t* sub_col = (ct == RAY_SYM) ? ray_sym_vec_new(full_col->attrs & RAY_SYM_W_MASK, gsize) : ray_vec_new(ct, gsize); - if (RAY_IS_ERR(sub_col)) { ray_release(sub_tbl); ray_release(out_col); ray_release(result); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return sub_col; } + if (RAY_IS_ERR(sub_col)) { ray_release(sub_tbl); ray_release(out_col); UPDATE_BY_CLEANUP_COLS(); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return sub_col; } /* per-group gather raw-copies cell ids from ONE * source column — keep its dictionary */ if (ct == RAY_SYM) @@ -11435,19 +11679,19 @@ ray_t* ray_update(ray_t** args, int64_t n) { memcpy(dst + r * esz, src + idxs[r] * esz, esz); sub_tbl = ray_table_add_col(sub_tbl, cn, sub_col); ray_release(sub_col); - if (RAY_IS_ERR(sub_tbl)) { ray_release(out_col); ray_release(result); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return sub_tbl; } + if (RAY_IS_ERR(sub_tbl)) { ray_release(out_col); UPDATE_BY_CLEANUP_COLS(); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return sub_tbl; } } /* Evaluate expression on sub-table via DAG */ ray_graph_t* ug = ray_graph_new(sub_tbl); ray_op_t* expr_op = compile_expr_dag(ug, agg_expr); - if (!expr_op) { ray_graph_free(ug); ray_release(sub_tbl); ray_release(out_col); ray_release(result); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return ray_error("domain", "update by: failed to compile aggregate expression"); } + if (!expr_op) { ray_graph_free(ug); ray_release(sub_tbl); ray_release(out_col); UPDATE_BY_CLEANUP_COLS(); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return ray_error("domain", "update by: failed to compile aggregate expression"); } expr_op = ray_optimize(ug, expr_op); ray_t* agg_result = ray_execute(ug, expr_op); ray_graph_free(ug); ray_release(sub_tbl); - if (RAY_IS_ERR(agg_result)) { ray_release(out_col); ray_release(result); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return agg_result; } + if (RAY_IS_ERR(agg_result)) { ray_release(out_col); UPDATE_BY_CLEANUP_COLS(); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return agg_result; } /* Determine output type from first group */ if (first_group) { @@ -11455,25 +11699,77 @@ ray_t* ray_update(ray_t** args, int64_t n) { else if (ray_is_vec(agg_result)) out_type = agg_result->type; ray_release(out_col); out_col = ray_vec_new(out_type, nrows2); - if (RAY_IS_ERR(out_col)) { ray_release(agg_result); ray_release(result); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return out_col; } + if (RAY_IS_ERR(out_col)) { ray_release(agg_result); UPDATE_BY_CLEANUP_COLS(); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return out_col; } out_col->len = nrows2; + memset(ray_data(out_col), 0, + (size_t)nrows2 * (size_t)ray_sym_elem_size(out_col->type, out_col->attrs)); first_group = 0; } - /* Broadcast aggregate value to all rows in this group */ + /* Scatter the group's result back to its rows. An atom + * broadcasts to every row of the group; a per-row vector + * (e.g. `update {v: (* v 2) by: k}`) scatters elementwise + * through idxs[r], symmetric with the atom branch. Any other + * shape — a vector whose length is neither 1 nor the group + * size — has no row-aligned meaning, so decline loudly rather + * than leave the memset zeros in place (silent data loss). */ int64_t* idxs = (int64_t*)ray_data(idx_vec); if (ray_is_atom(agg_result)) { for (int64_t r = 0; r < gsize; r++) store_typed_elem(out_col, idxs[r], agg_result); + } else if (ray_is_vec(agg_result) && ray_len(agg_result) == gsize) { + for (int64_t r = 0; r < gsize; r++) { + int alloc = 0; + ray_t* cell = collection_elem(agg_result, r, &alloc); + store_typed_elem(out_col, idxs[r], cell); + if (alloc) ray_release(cell); + } + } else { + int64_t got = ray_is_vec(agg_result) ? ray_len(agg_result) : -1; + ray_release(agg_result); ray_release(out_col); + UPDATE_BY_CLEANUP_COLS(); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); + return ray_error("length", "update by: expression result length %lld does not match group size %lld", (long long)got, (long long)gsize); } ray_release(agg_result); } - /* Add the new column to the result table */ - result = ray_table_add_col(result, kid, out_col); - ray_release(out_col); - if (RAY_IS_ERR(result)) { ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return result; } + upd_names[upd_i] = kid; + upd_cols[upd_i] = out_col; + upd_i++; + } + + /* Build result in schema order: replace existing targets in place, then + * append genuinely new update columns in dict order. */ + ray_t* result = ray_table_new((int32_t)(ncols + n_updates)); + if (RAY_IS_ERR(result)) { UPDATE_BY_CLEANUP_COLS(); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return result; } + for (int64_t c = 0; c < ncols; c++) { + int64_t cn = ray_table_col_name(tbl, c); + int64_t ui = -1; + for (int64_t u = 0; u < n_updates; u++) { + if (upd_names[u] == cn) { ui = u; break; } + } + if (ui >= 0) { + result = ray_table_add_col(result, cn, upd_cols[ui]); + ray_release(upd_cols[ui]); + upd_cols[ui] = NULL; + upd_used[ui] = 1; + } else { + ray_t* col = ray_table_get_col_idx(tbl, c); + ray_retain(col); + result = ray_table_add_col(result, cn, col); + ray_release(col); + } + if (RAY_IS_ERR(result)) { UPDATE_BY_CLEANUP_COLS(); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return result; } } + for (int64_t u = 0; u < n_updates; u++) { + if (upd_used[u]) continue; + result = ray_table_add_col(result, upd_names[u], upd_cols[u]); + ray_release(upd_cols[u]); + upd_cols[u] = NULL; + if (RAY_IS_ERR(result)) { UPDATE_BY_CLEANUP_COLS(); ray_release(groups); ray_release(tbl); DICT_VIEW_CLOSE(updv); return result; } + } + UPDATE_BY_CLEANUP_COLS(); + #undef UPDATE_BY_CLEANUP_COLS ray_release(groups); /* Store in-place and return the symbol if amending by name. */ @@ -14876,6 +15172,21 @@ static ray_t* window_join_impl(ray_t** args, int64_t n, int mode) { for (int i = 0; i < 4; i++) ray_release(eargs[i]); return ray_error("domain", "window-join: equality key column not found in both tables"); } + /* STR equality-key columns are silently mismatched: window-join + * sorts and probes eq cells through read_col_i64, which has no + * RAY_STR case (it reads one raw byte per row), so once there are + * enough distinct keys the byte collisions cross-contaminate + * groups and the aggregates are WRONG. Decline both sides — the + * base equi-join has a STR-aware kernel, these window kernels do + * not. */ + int8_t lct = left_eq[e]->type, rct = right_eq[e]->type; + if (RAY_IS_PARTED(lct)) lct = (int8_t)RAY_PARTED_BASETYPE(lct); + if (RAY_IS_PARTED(rct)) rct = (int8_t)RAY_PARTED_BASETYPE(rct); + if (lct == RAY_STR || rct == RAY_STR) { + scratch_free(eq_hdr); + for (int i = 0; i < 4; i++) ray_release(eargs[i]); + return ray_error("nyi", "window-join: string equality key columns are not supported"); + } } /* Parse every (name, (op src)) pair from the agg dict. The dict's @@ -15436,6 +15747,18 @@ static ray_t* window_join_impl(ray_t** args, int64_t n, int mode) { if (!nm) { scratch_free(eqops_hdr); ray_graph_free(g); if (_bxeq) ray_release(_bxeq); return ray_error("domain", "window-join: unknown equality key symbol"); } eq_ops[i] = ray_scan(g, ray_str_ptr(nm)); if (!eq_ops[i]) { scratch_free(eqops_hdr); ray_graph_free(g); if (_bxeq) ray_release(_bxeq); return ray_error("domain", "window-join: equality key column not found"); } + ray_t* lcol = ray_table_get_col(left_tbl, eq_elems[i]->i64); + ray_t* rcol = ray_table_get_col(right_tbl, eq_elems[i]->i64); + int8_t lct = lcol ? lcol->type : (int8_t)0; + int8_t rct = rcol ? rcol->type : (int8_t)0; + if (RAY_IS_PARTED(lct)) lct = (int8_t)RAY_PARTED_BASETYPE(lct); + if (RAY_IS_PARTED(rct)) rct = (int8_t)RAY_PARTED_BASETYPE(rct); + if (lct == RAY_STR || rct == RAY_STR) { + scratch_free(eqops_hdr); + ray_graph_free(g); + if (_bxeq) ray_release(_bxeq); + return ray_error("nyi", "window-join: string equality key columns are not supported"); + } } if (_bxeq) ray_release(_bxeq); @@ -15546,6 +15869,23 @@ static ray_t* ray_asof_join_core(ray_t* keys_vec, ray_t* left_tbl, ray_t* right_ if (!nm) { scratch_free(eqops_hdr); ray_graph_free(g); if (_bxk) ray_release(_bxk); return ray_error("domain", "asof-join: unknown equality key symbol"); } eq_ops[i] = ray_scan(g, ray_str_ptr(nm)); if (!eq_ops[i]) { scratch_free(eqops_hdr); ray_graph_free(g); if (_bxk) ray_release(_bxk); return ray_error("domain", "asof-join: equality key column not found"); } + /* STR equality-key columns are silently mismatched: the asof kernel + * reads eq cells through read_col_i64 (asof_eq_lread), which has no + * RAY_STR case, so string keys compare as raw bytes and yield WRONG + * results (a key that should match can null out). The base equi-join + * has a separate STR-aware kernel; asof does not, so decline both + * sides rather than return corrupt data. */ + ray_t* lcol = ray_table_get_col(left_tbl, eq_syms[i]->i64); + ray_t* rcol = ray_table_get_col(right_tbl, eq_syms[i]->i64); + int8_t lct = lcol ? lcol->type : (int8_t)0; + int8_t rct = rcol ? rcol->type : (int8_t)0; + if (RAY_IS_PARTED(lct)) lct = (int8_t)RAY_PARTED_BASETYPE(lct); + if (RAY_IS_PARTED(rct)) rct = (int8_t)RAY_PARTED_BASETYPE(rct); + if (lct == RAY_STR || rct == RAY_STR) { + scratch_free(eqops_hdr); + ray_graph_free(g); if (_bxk) ray_release(_bxk); + return ray_error("nyi", "asof-join: string equality key columns are not supported"); + } } if (_bxk) ray_release(_bxk); @@ -16044,7 +16384,7 @@ static ray_t* try_stream_parted_order_by_topk( /* Final merge: apply_sort_take reads asc/desc/take straight from dict_elems * and is the SAME stable kernel the flat path uses, so the k rows and their * order are byte-identical to flatten→sort→take. It consumes `accum`. */ - return apply_sort_take(accum, dict_elems, dict_n, asc_id, desc_id, take_id); + return apply_sort_take(accum, dict_elems, dict_n, asc_id, desc_id, take_id, NULL); } /* If `tbl` is DATE-partitioned, return its (borrowed) per-partition RAY_DATE diff --git a/src/ops/temporal.c b/src/ops/temporal.c index 9ea07c88..9a6ab254 100644 --- a/src/ops/temporal.c +++ b/src/ops/temporal.c @@ -25,6 +25,7 @@ #include "lang/internal.h" #include "ops/temporal.h" #include "lang/format.h" /* ray_type_name (error context) */ +#include "core/pool.h" /* ray_pool_dispatch — parallel extract/truncate */ #include /* ============================================================================ @@ -163,6 +164,113 @@ static inline bool rte_trunc_elem(int8_t t, int64_t raw, int64_t bucket, int64_t return true; } +/* ---------------------------------------------------------------------------- + * Parallel driver for the two whole-column kernels below. + * + * Both loops are pure elementwise maps: row i of the output depends only on + * row i of the input, so any partition of [0, len) is safe. The ONE piece of + * cross-row state is the null marking. Nulls in this engine are *sentinels in + * the payload* (see ray_vec_set_null_checked) — the per-row write is already + * disjoint — but ray_vec_set_null also does a read-modify-write of the shared + * `attrs` byte (RAY_ATTR_HAS_NULLS, and the SORTED clear inside + * vec_drop_index_inplace). That byte is what races, not the bitmap the + * bitmap-based engines would have. + * + * So the workers never touch `attrs`: each writes the NULL_* sentinel into its + * own rows and reports "I produced at least one null" through one atomic flag, + * and the caller folds that into `result->attrs` once, single-threaded, after + * the dispatch has joined. Result is byte-identical to the serial version: + * the destination is a fresh ray_vec_new (attrs == 0, no index, not a slice), + * so ray_vec_set_null on it reduces to exactly "sentinel + HAS_NULLS". + * -------------------------------------------------------------------------- */ +typedef struct { + ray_t* input; + const char* base; + int64_t* out; + int64_t bucket; /* truncate only */ + int field; + int8_t t; + bool src_has_nulls; + bool in32; /* RAY_DATE / RAY_TIME element is int32 */ + _Atomic(uint32_t) any_null; +} rte_par_ctx_t; + +#define RTE_RANGE_BODY(CALL) \ + do { \ + if (c->in32) { \ + const int32_t* d32 = (const int32_t*)c->base; \ + if (c->src_has_nulls) { \ + for (int64_t i = start; i < end; i++) \ + if (ray_vec_is_null(c->input, i) || \ + !CALL((int64_t)d32[i])) { \ + c->out[i] = NULL_I64; nulled = true; \ + } \ + } else { \ + for (int64_t i = start; i < end; i++) \ + if (!CALL((int64_t)d32[i])) { \ + c->out[i] = NULL_I64; nulled = true; \ + } \ + } \ + } else { \ + const int64_t* d64 = (const int64_t*)c->base; \ + if (c->src_has_nulls) { \ + for (int64_t i = start; i < end; i++) \ + if (ray_vec_is_null(c->input, i) || !CALL(d64[i])) { \ + c->out[i] = NULL_I64; nulled = true; \ + } \ + } else { \ + for (int64_t i = start; i < end; i++) \ + if (!CALL(d64[i])) { \ + c->out[i] = NULL_I64; nulled = true; \ + } \ + } \ + } \ + if (nulled) \ + atomic_store_explicit(&c->any_null, 1, memory_order_relaxed); \ + } while (0) + +#define RTE_EXTRACT_CALL(RAW) rte_extract_elem(c->t, (RAW), c->field, &c->out[i]) +#define RTE_TRUNC_CALL(RAW) rte_trunc_elem(c->t, (RAW), c->bucket, &c->out[i]) + +static void rte_extract_range(rte_par_ctx_t* c, int64_t start, int64_t end) { + bool nulled = false; + RTE_RANGE_BODY(RTE_EXTRACT_CALL); +} + +static void rte_trunc_range(rte_par_ctx_t* c, int64_t start, int64_t end) { + bool nulled = false; + RTE_RANGE_BODY(RTE_TRUNC_CALL); +} + +#undef RTE_EXTRACT_CALL +#undef RTE_TRUNC_CALL +#undef RTE_RANGE_BODY + +static void rte_extract_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t end) { + (void)worker_id; + rte_extract_range((rte_par_ctx_t*)ctx, start, end); +} + +static void rte_trunc_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t end) { + (void)worker_id; + rte_trunc_range((rte_par_ctx_t*)ctx, start, end); +} + +/* Run `range_fn` over [0, len), in parallel when the column is big enough to + * amortize dispatch (the engine-wide RAY_PARALLEL_THRESHOLD, same gate + * expr_eval_full uses — no new knob), then fold the null flag into `result`. */ +static void rte_run(rte_par_ctx_t* c, ray_pool_fn task_fn, + void (*range_fn)(rte_par_ctx_t*, int64_t, int64_t), + ray_t* result, int64_t len) { + ray_pool_t* pool = ray_pool_get(); + if (ray_pool_par_dispatch_ok(pool, len, RAY_PARALLEL_THRESHOLD)) + ray_pool_dispatch(pool, task_fn, c, len); + else + range_fn(c, 0, len); + if (atomic_load_explicit(&c->any_null, memory_order_relaxed)) + result->attrs |= RAY_ATTR_HAS_NULLS; +} + ray_t* ray_temporal_extract(ray_t* input, int field) { if (!input || RAY_IS_ERR(input)) return input; @@ -202,44 +310,20 @@ ray_t* ray_temporal_extract(ray_t* input, int field) { (input->attrs & RAY_ATTR_HAS_NULLS) || ((input->attrs & RAY_ATTR_SLICE) && input->slice_parent && (input->slice_parent->attrs & RAY_ATTR_HAS_NULLS)); - const char* base = (const char*)ray_data(input); - /* Hoist src_has_nulls and type dispatch outside the loop so the - * inner body is a tight typed kernel with no per-element branches. */ - if (t == RAY_DATE || t == RAY_TIME) { - const int32_t* d32 = (const int32_t*)base; - if (src_has_nulls) { - for (int64_t i = 0; i < len; i++) { - if (ray_vec_is_null(input, i) || - !rte_extract_elem(t, (int64_t)d32[i], field, &out[i])) { - out[i] = NULL_I64; - ray_vec_set_null(result, i, true); - } - } - } else { - for (int64_t i = 0; i < len; i++) - if (!rte_extract_elem(t, (int64_t)d32[i], field, &out[i])) { - out[i] = NULL_I64; - ray_vec_set_null(result, i, true); - } - } - } else { - const int64_t* d64 = (const int64_t*)base; - if (src_has_nulls) { - for (int64_t i = 0; i < len; i++) { - if (ray_vec_is_null(input, i) || - !rte_extract_elem(t, d64[i], field, &out[i])) { - out[i] = NULL_I64; - ray_vec_set_null(result, i, true); - } - } - } else { - for (int64_t i = 0; i < len; i++) - if (!rte_extract_elem(t, d64[i], field, &out[i])) { - out[i] = NULL_I64; - ray_vec_set_null(result, i, true); - } - } - } + /* src_has_nulls and the 32-/64-bit element dispatch are hoisted into the + * context so the inner body is a tight typed kernel with no per-element + * branches; the row range is chunked over the pool for large columns. */ + rte_par_ctx_t c = { + .input = input, + .base = (const char*)ray_data(input), + .out = out, + .field = field, + .t = t, + .src_has_nulls = src_has_nulls, + .in32 = (t == RAY_DATE || t == RAY_TIME), + .any_null = 0, + }; + rte_run(&c, rte_extract_fn, rte_extract_range, result, len); return result; } @@ -338,36 +422,18 @@ ray_t* ray_temporal_truncate(ray_t* input, int kind) { ? RTE_USEC_PER_DAY : RTE_USEC_PER_SEC; - /* Hoist src_has_nulls and type dispatch outside the loop. */ - if (t == RAY_DATE || t == RAY_TIME) { - const int32_t* d32 = (const int32_t*)base; - if (src_has_nulls) { - for (int64_t i = 0; i < len; i++) - if (ray_vec_is_null(input, i) || - !rte_trunc_elem(t, (int64_t)d32[i], bucket, &out[i])) { - out[i] = NULL_I64; ray_vec_set_null(result, i, true); - } - } else { - for (int64_t i = 0; i < len; i++) - if (!rte_trunc_elem(t, (int64_t)d32[i], bucket, &out[i])) { - out[i] = NULL_I64; ray_vec_set_null(result, i, true); - } - } - } else { - const int64_t* d64 = (const int64_t*)base; - if (src_has_nulls) { - for (int64_t i = 0; i < len; i++) - if (ray_vec_is_null(input, i) || - !rte_trunc_elem(t, d64[i], bucket, &out[i])) { - out[i] = NULL_I64; ray_vec_set_null(result, i, true); - } - } else { - for (int64_t i = 0; i < len; i++) - if (!rte_trunc_elem(t, d64[i], bucket, &out[i])) { - out[i] = NULL_I64; ray_vec_set_null(result, i, true); - } - } - } + /* Same hoist-and-chunk shape as ray_temporal_extract above. */ + rte_par_ctx_t c = { + .input = input, + .base = base, + .out = out, + .bucket = bucket, + .t = t, + .src_has_nulls = src_has_nulls, + .in32 = (t == RAY_DATE || t == RAY_TIME), + .any_null = 0, + }; + rte_run(&c, rte_trunc_fn, rte_trunc_range, result, len); return result; } @@ -382,35 +448,32 @@ ray_t* ray_temporal_truncate(ray_t* input, int kind) { * Gregorian calendar decomposition. * ============================================================================ */ -ray_t* exec_extract(ray_graph_t* g, ray_op_t* op) { - ray_t* input = exec_node(g, op_child(g, op, 0)); - if (!input || RAY_IS_ERR(input)) return input; - - ray_op_ext_t* ext = find_ext(g, op->id); - if (!ext) { ray_release(input); return ray_error("nyi", NULL); } - - int64_t field = ext->sym; - int64_t len = input->len; - int8_t in_type = input->type; - - ray_t* result = ray_vec_new(RAY_I64, len); - if (!result || RAY_IS_ERR(result)) { ray_release(input); return result; } - result->len = len; - - int64_t* out = (int64_t*)ray_data(result); - - #undef USEC_PER_SEC - #define USEC_PER_SEC 1000000LL - #define USEC_PER_MIN (60LL * USEC_PER_SEC) - #define USEC_PER_HOUR (3600LL * USEC_PER_SEC) - #define USEC_PER_DAY (86400LL * USEC_PER_SEC) - - /* Slice-aware HAS_NULLS check: slices don't carry HAS_NULLS on - * themselves, so inspect the parent when input is a slice. */ - bool src_has_nulls = - (input->attrs & RAY_ATTR_HAS_NULLS) || - ((input->attrs & RAY_ATTR_SLICE) && input->slice_parent && - (input->slice_parent->attrs & RAY_ATTR_HAS_NULLS)); +/* Per-worker state for the DAG-level extract. Same contract as + * rte_par_ctx_t above: workers own disjoint row ranges, write NULL_I64 + * sentinels themselves, and report the HAS_NULLS decision through one + * atomic flag the caller folds into `result` after the join. */ +typedef struct { + ray_t* input; + int64_t* out; + int64_t field; + int8_t in_type; + bool src_has_nulls; + bool in32; + _Atomic(uint32_t) any_null; +} xtr_par_ctx_t; + +#undef USEC_PER_SEC +#define USEC_PER_SEC 1000000LL +#define USEC_PER_MIN (60LL * USEC_PER_SEC) +#define USEC_PER_HOUR (3600LL * USEC_PER_SEC) +#define USEC_PER_DAY (86400LL * USEC_PER_SEC) + +static void xtr_extract_range(xtr_par_ctx_t* c, int64_t start, int64_t end) { + ray_t* input = c->input; + int64_t* out = c->out; + int64_t field = c->field; + int8_t in_type = c->in_type; + bool nulled = false; /* Macro to emit a tight inner loop body with loop-invariant branches * hoisted at compile time. HAS_NULLS and IN32 are 0/1 constants so @@ -423,7 +486,7 @@ ray_t* exec_extract(ray_graph_t* g, ray_op_t* op) { for (int64_t i = 0; i < n; i++) { \ if (HAS_NULLS && ray_vec_is_null(input, off + i)) { \ out[off + i] = NULL_I64; \ - ray_vec_set_null(result, off + i, true); \ + nulled = true; \ continue; \ } \ int64_t us; \ @@ -437,7 +500,7 @@ ray_t* exec_extract(ray_graph_t* g, ray_op_t* op) { ((int64_t)raw32 > INT64_MAX / USEC_PER_DAY || \ (int64_t)raw32 < INT64_MIN / USEC_PER_DAY)) { \ out[off + i] = NULL_I64; \ - ray_vec_set_null(result, off + i, true); \ + nulled = true; \ continue; \ } \ us = (in_type == RAY_DATE) \ @@ -506,12 +569,13 @@ ray_t* exec_extract(ray_graph_t* g, ray_op_t* op) { } while (0) ray_morsel_t m; - ray_morsel_init(&m, input); - int64_t off = 0; + ray_morsel_init_range(&m, input, start, end); + int64_t off = start; /* Hoist src_has_nulls and in_type (32- vs 64-bit element) outside * all loops using the macro dispatch above. */ - bool in32 = (in_type == RAY_DATE || in_type == RAY_TIME); + const bool src_has_nulls = c->src_has_nulls; + const bool in32 = c->in32; if (!src_has_nulls && !in32) EXTRACT_INNER(0, 0); else if (!src_has_nulls && in32) EXTRACT_INNER(0, 1); else if ( src_has_nulls && !in32) EXTRACT_INNER(1, 0); @@ -519,10 +583,65 @@ ray_t* exec_extract(ray_graph_t* g, ray_op_t* op) { #undef EXTRACT_INNER - #undef USEC_PER_SEC - #undef USEC_PER_MIN - #undef USEC_PER_HOUR - #undef USEC_PER_DAY + if (nulled) atomic_store_explicit(&c->any_null, 1, memory_order_relaxed); +} + +#undef USEC_PER_SEC +#undef USEC_PER_MIN +#undef USEC_PER_HOUR +#undef USEC_PER_DAY + +static void xtr_extract_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t end) { + (void)worker_id; + xtr_extract_range((xtr_par_ctx_t*)ctx, start, end); +} + +ray_t* exec_extract(ray_graph_t* g, ray_op_t* op) { + ray_t* input = exec_node(g, op_child(g, op, 0)); + if (!input || RAY_IS_ERR(input)) return input; + + ray_op_ext_t* ext = find_ext(g, op->id); + if (!ext) { ray_release(input); return ray_error("nyi", NULL); } + + int64_t field = ext->sym; + int64_t len = input->len; + int8_t in_type = input->type; + + ray_t* result = ray_vec_new(RAY_I64, len); + if (!result || RAY_IS_ERR(result)) { ray_release(input); return result; } + result->len = len; + + /* Slice-aware HAS_NULLS check: slices don't carry HAS_NULLS on + * themselves, so inspect the parent when input is a slice. */ + bool src_has_nulls = + (input->attrs & RAY_ATTR_HAS_NULLS) || + ((input->attrs & RAY_ATTR_SLICE) && input->slice_parent && + (input->slice_parent->attrs & RAY_ATTR_HAS_NULLS)); + + /* Preserve ray_morsel_init's one-time sequential-readahead hint for + * mmap'd columns. The per-worker ray_morsel_init_range deliberately + * does not issue it — it would fire once per task instead of once per + * column — so it is issued here, on the main thread, before dispatch. */ + ray_morsel_t warm; + ray_morsel_init(&warm, input); + + xtr_par_ctx_t c = { + .input = input, + .out = (int64_t*)ray_data(result), + .field = field, + .in_type = in_type, + .src_has_nulls = src_has_nulls, + .in32 = (in_type == RAY_DATE || in_type == RAY_TIME), + .any_null = 0, + }; + + ray_pool_t* pool = ray_pool_get(); + if (ray_pool_par_dispatch_ok(pool, len, RAY_PARALLEL_THRESHOLD)) + ray_pool_dispatch(pool, xtr_extract_fn, &c, len); + else + xtr_extract_range(&c, 0, len); + if (atomic_load_explicit(&c.any_null, memory_order_relaxed)) + result->attrs |= RAY_ATTR_HAS_NULLS; ray_release(input); return result; @@ -548,34 +667,17 @@ static int64_t days_from_civil(int64_t y, int64_t m, int64_t d) { return era * 146097 + (int64_t)doe - 719468 - 10957; } -ray_t* exec_date_trunc(ray_graph_t* g, ray_op_t* op) { - ray_t* input = exec_node(g, op_child(g, op, 0)); - if (!input || RAY_IS_ERR(input)) return input; - - ray_op_ext_t* ext = find_ext(g, op->id); - if (!ext) { ray_release(input); return ray_error("nyi", NULL); } - - int64_t field = ext->sym; - int64_t len = input->len; - int8_t in_type = input->type; +#define DT_USEC_PER_SEC 1000000LL +#define DT_USEC_PER_MIN (60LL * DT_USEC_PER_SEC) +#define DT_USEC_PER_HOUR (3600LL * DT_USEC_PER_SEC) +#define DT_USEC_PER_DAY (86400LL * DT_USEC_PER_SEC) - ray_t* result = ray_vec_new(RAY_TIMESTAMP, len); - if (!result || RAY_IS_ERR(result)) { ray_release(input); return result; } - result->len = len; - - int64_t* out = (int64_t*)ray_data(result); - - #define DT_USEC_PER_SEC 1000000LL - #define DT_USEC_PER_MIN (60LL * DT_USEC_PER_SEC) - #define DT_USEC_PER_HOUR (3600LL * DT_USEC_PER_SEC) - #define DT_USEC_PER_DAY (86400LL * DT_USEC_PER_SEC) - - /* Slice-aware HAS_NULLS check: slices don't carry HAS_NULLS on - * themselves, so inspect the parent when input is a slice. */ - bool src_has_nulls = - (input->attrs & RAY_ATTR_HAS_NULLS) || - ((input->attrs & RAY_ATTR_SLICE) && input->slice_parent && - (input->slice_parent->attrs & RAY_ATTR_HAS_NULLS)); +static void xtr_trunc_range(xtr_par_ctx_t* c, int64_t start, int64_t end) { + ray_t* input = c->input; + int64_t* out = c->out; + int64_t field = c->field; + int8_t in_type = c->in_type; + bool nulled = false; /* Macro to emit a tight inner loop body with HAS_NULLS and IN32 hoisted * as compile-time constants (DCE removes dead branches). @@ -587,7 +689,7 @@ ray_t* exec_date_trunc(ray_graph_t* g, ray_op_t* op) { for (int64_t i = 0; i < n; i++) { \ if (HAS_NULLS && ray_vec_is_null(input, off + i)) { \ out[off + i] = NULL_I64; \ - ray_vec_set_null(result, off + i, true); \ + nulled = true; \ continue; \ } \ int64_t us; \ @@ -604,7 +706,7 @@ ray_t* exec_date_trunc(ray_graph_t* g, ray_op_t* op) { ((int64_t)raw32 > INT64_MAX / 1000 / DT_USEC_PER_DAY || \ (int64_t)raw32 < INT64_MIN / 1000 / DT_USEC_PER_DAY)) { \ out[off + i] = NULL_I64; \ - ray_vec_set_null(result, off + i, true); \ + nulled = true; \ continue; \ } \ us = (in_type == RAY_DATE) \ @@ -676,7 +778,7 @@ ray_t* exec_date_trunc(ray_graph_t* g, ray_op_t* op) { /* Result must fit int64 nanoseconds (out_us × 1000). */ \ if (out_us > INT64_MAX / 1000LL || out_us < INT64_MIN / 1000LL) { \ out[off + i] = NULL_I64; \ - ray_vec_set_null(result, off + i, true); \ + nulled = true; \ continue; \ } \ out[off + i] = out_us * 1000LL; /* µs → ns for RAY_TIMESTAMP */ \ @@ -686,11 +788,12 @@ ray_t* exec_date_trunc(ray_graph_t* g, ray_op_t* op) { } while (0) ray_morsel_t m; - ray_morsel_init(&m, input); - int64_t off = 0; + ray_morsel_init_range(&m, input, start, end); + int64_t off = start; /* Hoist src_has_nulls and in_type dispatch outside all loops. */ - bool dt_in32 = (in_type == RAY_DATE || in_type == RAY_TIME); + const bool src_has_nulls = c->src_has_nulls; + const bool dt_in32 = c->in32; if (!src_has_nulls && !dt_in32) DATE_TRUNC_INNER(0, 0); else if (!src_has_nulls && dt_in32) DATE_TRUNC_INNER(0, 1); else if ( src_has_nulls && !dt_in32) DATE_TRUNC_INNER(1, 0); @@ -698,10 +801,65 @@ ray_t* exec_date_trunc(ray_graph_t* g, ray_op_t* op) { #undef DATE_TRUNC_INNER - #undef DT_USEC_PER_SEC - #undef DT_USEC_PER_MIN - #undef DT_USEC_PER_HOUR - #undef DT_USEC_PER_DAY + if (nulled) atomic_store_explicit(&c->any_null, 1, memory_order_relaxed); +} + +#undef DT_USEC_PER_SEC +#undef DT_USEC_PER_MIN +#undef DT_USEC_PER_HOUR +#undef DT_USEC_PER_DAY + +static void xtr_trunc_fn(void* ctx, uint32_t worker_id, int64_t start, int64_t end) { + (void)worker_id; + xtr_trunc_range((xtr_par_ctx_t*)ctx, start, end); +} + +ray_t* exec_date_trunc(ray_graph_t* g, ray_op_t* op) { + ray_t* input = exec_node(g, op_child(g, op, 0)); + if (!input || RAY_IS_ERR(input)) return input; + + ray_op_ext_t* ext = find_ext(g, op->id); + if (!ext) { ray_release(input); return ray_error("nyi", NULL); } + + int64_t field = ext->sym; + int64_t len = input->len; + int8_t in_type = input->type; + + ray_t* result = ray_vec_new(RAY_TIMESTAMP, len); + if (!result || RAY_IS_ERR(result)) { ray_release(input); return result; } + result->len = len; + + /* Slice-aware HAS_NULLS check: slices don't carry HAS_NULLS on + * themselves, so inspect the parent when input is a slice. */ + bool src_has_nulls = + (input->attrs & RAY_ATTR_HAS_NULLS) || + ((input->attrs & RAY_ATTR_SLICE) && input->slice_parent && + (input->slice_parent->attrs & RAY_ATTR_HAS_NULLS)); + + /* Preserve ray_morsel_init's one-time sequential-readahead hint for + * mmap'd columns. The per-worker ray_morsel_init_range deliberately + * does not issue it — it would fire once per task instead of once per + * column — so it is issued here, on the main thread, before dispatch. */ + ray_morsel_t warm; + ray_morsel_init(&warm, input); + + xtr_par_ctx_t c = { + .input = input, + .out = (int64_t*)ray_data(result), + .field = field, + .in_type = in_type, + .src_has_nulls = src_has_nulls, + .in32 = (in_type == RAY_DATE || in_type == RAY_TIME), + .any_null = 0, + }; + + ray_pool_t* pool = ray_pool_get(); + if (ray_pool_par_dispatch_ok(pool, len, RAY_PARALLEL_THRESHOLD)) + ray_pool_dispatch(pool, xtr_trunc_fn, &c, len); + else + xtr_trunc_range(&c, 0, len); + if (atomic_load_explicit(&c.any_null, memory_order_relaxed)) + result->attrs |= RAY_ATTR_HAS_NULLS; ray_release(input); return result; diff --git a/src/store/part.c b/src/store/part.c index ebe0ce15..6bc5dd69 100644 --- a/src/store/part.c +++ b/src/store/part.c @@ -27,6 +27,7 @@ #define _GNU_SOURCE #endif #include "part.h" +#include "core/runtime.h" #include "mem/sys.h" #include "ops/ops.h" #include "store/splay.h" @@ -560,7 +561,21 @@ static ray_t* empty_table_like(ray_t* tmpl) { for (int64_t c = 0; c < ncols; c++) { ray_t* col = ray_table_get_col_idx(tmpl, c); if (!col) { ray_release(out); return ray_error("type", "empty_table_like: null column"); } - ray_t* ecol = ray_vec_new(col->type, 0); + ray_t* ecol = NULL; + if (col->type == RAY_LIST) { + ecol = ray_list_new(0); + } else if (col->type == RAY_TABLE || col->type == RAY_DICT) { + int64_t name_id = ray_table_col_name(tmpl, c); + ray_t* name = ray_sym_str(name_id); + const char* name_ptr = name ? ray_str_ptr(name) : "?"; + int name_len = name ? (int)ray_str_len(name) : 1; + const char* type_name = col->type == RAY_TABLE ? "TABLE" : "DICT"; + ray_release(out); + return ray_error("nyi", "empty_table_like: column '%.*s' has unsupported nested type %s", + name_len, name_ptr, type_name); + } else { + ecol = ray_vec_new(col->type, 0); + } if (!ecol || RAY_IS_ERR(ecol)) { ray_release(out); return ecol ? ecol : ray_error("oom", NULL); } ray_t* nout = ray_table_add_col(out, ray_table_col_name(tmpl, c), ecol); ray_release(ecol); @@ -674,6 +689,7 @@ ray_t* ray_parted_fill(const char* db_root) { int64_t all_count = 0, all_cap = 0; uint8_t* fixed = (uint8_t*)ray_calloc_raw((size_t)((size_t)part_count) * (1)); ray_err_t err = fixed ? RAY_OK : RAY_ERR_OOM; + char cause[256] = {0}; for (int64_t p = 0; p < part_count && err == RAY_OK; p++) { char pdir[1024]; @@ -718,16 +734,23 @@ ray_t* ray_parted_fill(const char* db_root) { if (tn < 0 || (size_t)tn >= sizeof(tdir)) { err = RAY_ERR_RANGE; break; } ray_t* full = ray_read_splayed(tdir, sym_path); if (!full || RAY_IS_ERR(full)) { + err = full ? ray_err_from_obj(full) : RAY_ERR_OOM; + const char* detail = ray_error_msg(); + snprintf(cause, sizeof(cause), "table %s (partition %s): %s", + tname, part_dirs[templ], + detail && *detail ? detail : "template read failed"); if (full) ray_error_free(full); - err = RAY_ERR_IO; break; } empty_tbl = empty_table_like(full); ray_release(full); if (!empty_tbl || RAY_IS_ERR(empty_tbl)) { + err = empty_tbl ? ray_err_from_obj(empty_tbl) : RAY_ERR_OOM; + const char* detail = ray_error_msg(); + snprintf(cause, sizeof(cause), "table %s: %s", tname, + detail && *detail ? detail : "empty table construction failed"); if (empty_tbl) ray_error_free(empty_tbl); empty_tbl = NULL; - err = RAY_ERR_IO; break; } } @@ -737,7 +760,12 @@ ray_t* ray_parted_fill(const char* db_root) { db_root, part_dirs[p], tname); if (ptn < 0 || (size_t)ptn >= sizeof(ptdir)) { err = RAY_ERR_RANGE; break; } ray_err_t se = ray_splay_save(empty_tbl, ptdir, sym_path); - if (se != RAY_OK) { err = se; break; } + if (se != RAY_OK) { + err = se; + snprintf(cause, sizeof(cause), "table %s (partition %s): save failed", + tname, part_dirs[p]); + break; + } fixed[p] = 1; } if (empty_tbl) ray_release(empty_tbl); @@ -767,6 +795,9 @@ ray_t* ray_parted_fill(const char* db_root) { if (err != RAY_OK) { if (result && !RAY_IS_ERR(result)) ray_release(result); + if (cause[0]) + return ray_error(ray_err_code_str(err), "parted %s: fill failed: %s", + db_root, cause); return ray_error(ray_err_code_str(err), "parted %s: fill failed", db_root); } return result; diff --git a/src/store/splay.c b/src/store/splay.c index 8e1acd5b..4350c64c 100644 --- a/src/store/splay.c +++ b/src/store/splay.c @@ -30,6 +30,7 @@ #include "table/table.h" #include "table/domain.h" #include "ops/idxop.h" +#include "io/csv.h" /* ray_csv_hash_upgrade_check — shared index policy */ #include "vec/str.h" #include "lang/format.h" #include @@ -560,12 +561,43 @@ void ray_splay_build_indexes(const char* dir, ray_t* tbl) { if (col->len < (1 << 16)) continue; /* STR columns get a dictionary (group on int codes); numeric/temporal - * get the per-chunk min/max for block-skip. */ + * get the per-chunk min/max for block-skip. + * + * High-entropy numeric columns are then upgraded to a persisted HASH + * index using the same chunk-zone entropy heuristic the in-memory + * .csv.read load applies (ray_csv_hash_upgrade_check): the index + * decision is made ONCE at conversion time, so a `.db.splayed.get` + * reload serves equality probes exactly as fast as a fresh CSV load + * (previously the reloaded store had only the zone map and e.g. + * ClickBench q10 ran 5x slower on-disk than in-memory). */ ray_t* idx = (col->type == RAY_STR) ? ray_index_dict_compute(col) : ray_index_chunk_zone_compute(col, 16); if (!idx || RAY_IS_ERR(idx)) { if (idx) ray_error_free(idx); continue; } + if (col->type != RAY_STR && + ray_csv_hash_upgrade_check(col->type, col->len, + ray_index_payload(idx))) { + ray_t* hi = ray_idx_hash_fn(col); + if (hi && !RAY_IS_ERR(hi) && (hi->attrs & RAY_ATTR_HAS_INDEX)) { + ray_release(idx); /* zone sacrificed for the hash */ + idx = NULL; + ray_t* nstr = ray_sym_str(ray_table_col_name(tbl, c)); + if (nstr && !RAY_IS_ERR(nstr)) { + char path[1100]; + int n = snprintf(path, sizeof(path), "%s/%.*s", dir, + (int)ray_str_len(nstr), ray_str_ptr(nstr)); + if (n > 0 && n < (int)sizeof(path)) + (void)ray_col_append_index(path, + ray_index_payload(hi->index), hi->len, hi->type); + } + ray_release(hi); + continue; + } + if (hi) { if (RAY_IS_ERR(hi)) ray_error_free(hi); else ray_release(hi); } + /* Hash build failed — fall through and persist the zone. */ + } + ray_t* nstr = ray_sym_str(ray_table_col_name(tbl, c)); if (nstr && !RAY_IS_ERR(nstr)) { char path[1100]; diff --git a/test/rfl/collection/drop_cut_rotate_cross.rfl b/test/rfl/collection/drop_cut_rotate_cross.rfl index bbf5a387..74b9538b 100644 --- a/test/rfl/collection/drop_cut_rotate_cross.rfl +++ b/test/rfl/collection/drop_cut_rotate_cross.rfl @@ -4,6 +4,9 @@ (drop [1 2 3 4 5] -2) -- [1 2 3] (drop [1 2 3] 10) -- [] (drop [1 2 3] -10) -- [] +;; INT64_MIN count: `-INT64_MIN` is signed-overflow UB (was tripped in +;; ray_drop_fn); |n| that large drops the whole collection. +(drop [1 2 3] -9223372036854775808) -- [] (drop "abcdef" 2) -- "cdef" (drop "abcdef" -2) -- "abcd" (drop ['a 'b 'c] 1) -- ['b 'c] diff --git a/test/rfl/collection/take.rfl b/test/rfl/collection/take.rfl index 92e1ef1d..bb1a954a 100644 --- a/test/rfl/collection/take.rfl +++ b/test/rfl/collection/take.rfl @@ -175,3 +175,12 @@ (count (take [1 2 3 4 5] [2 0])) -- 0 ;; range take with start = count → empty (count (take [1 2 3 4 5] [5 3])) -- 0 +;; INT64_MIN count: `-INT64_MIN` is signed-overflow UB (was tripped computing +;; the magnitude in every scalar-count take branch — vector, string, char, +;; scalar, list). Its magnitude is unrepresentable as int64, so it is a range +;; error, not UB followed by a bogus allocation. +(take [1 2 3] -9223372036854775808) !- range +(take "hello" -9223372036854775808) !- range +(take "a" -9223372036854775808) !- range +(take 42 -9223372036854775808) !- range +(take (list 1 2 3) -9223372036854775808) !- range diff --git a/test/rfl/group/radix_key_emit_parallel.rfl b/test/rfl/group/radix_key_emit_parallel.rfl new file mode 100644 index 00000000..8056f458 --- /dev/null +++ b/test/rfl/group/radix_key_emit_parallel.rfl @@ -0,0 +1,100 @@ +;; ════════════════════════════════════════════════════════════════════ +;; radix_key_emit_parallel.rfl — parallel key-column emit of the v2 radix +;; group-by (agg_key_emit_range / agg_key_emit_fn, src/ops/agg_engine.c). +;; +;; The emit un-packs each group's representative key tuple out of the +;; per-partition packed `keys` buffers into the result key columns, in +;; stable first-seen order. It used to run single-threaded over every +;; output row; it is now dispatched over disjoint output-row ranges when +;; n_emit >= RAY_PARALLEL_THRESHOLD (65536). +;; +;; What each fixture pins: +;; 1. WIDE-SYM — 200k singleton groups on a W32 SYM key: the widest +;; emit shape (and the one the 100M q13 profile is about). 200k is +;; enough to clear the dispatch threshold AND still land W32 SYM; +;; every group is a singleton in til() order, so the emitted key +;; column must equal the SOURCE column element-for-element — a +;; per-row oracle, not a spot check. +;; 2. WIDE-KEY — 17 keys of MIXED width (I64/I32/I16/U8/BOOL/DATE/ +;; TIME/TIMESTAMP/SYM + I64 constants) at 100k groups. Two things +;; at once: the packed-key stride (a per-column index error shows up +;; as a shifted column) and write_col_i64's per-type store width. +;; >16 keys ALSO takes the radix path deliberately — the key count is +;; unbounded there (only dense direct-index routing caps it at 16), +;; which is why the emit's destination-handle array must not be a +;; fixed [16] stack array. +;; 3. TAKE-BOUNDED — two shapes. RSB (single-key take: 10) is the +;; HEAD(GROUP) limit hint (q17's shape): n_emit is 10, far below the +;; dispatch threshold, so it pins the serial range call. RT (17-key +;; take: 10) does NOT get the pushdown — the limit hint doesn't fire +;; on this multi-key shape, so it emits all 100k rows in parallel and +;; apply_sort_take trims after; it pins the parallel emit + trim +;; composition, not the serial path. +;; +;; Routing (all three): >=1 int/SYM key with no nulls, >= 65536 rows, only +;; streaming aggs, and a key range too sparse for the parallel dense plan +;; (total_slots > rows/workers) -> exec_group_v2_parallel_radix. +;; ════════════════════════════════════════════════════════════════════ + +;; ---------- 1. wide-SYM key, 200k singleton groups ---------- +(set NS 200000) +(set KS (as 'SYM (til NS))) +(set TS (table [k v] (list KS (til NS)))) +(set RS (select {c: (count v) s: (sum v) from: TS by: k})) +(count RS) -- 200000 +;; singleton groups in first-seen order => the key column IS the source column +(all (== (at RS 'k) KS)) -- true +(all (== (at RS 'c) (+ 1 (* 0 (til NS))))) -- true +(all (== (at RS 's) (til NS))) -- true + +;; ---------- 2. seventeen keys, mixed widths, 100k groups ---------- +(set N 100000) +(set K0 (til N)) +(set K1 (as 'I32 (% (til N) 1000))) +(set K2 (as 'I16 (% (til N) 300))) +(set K3 (as 'U8 (% (til N) 251))) +(set K4 (as 'BOOL (% (til N) 2))) +(set K5 (as 'DATE (% (til N) 4000))) +(set K6 (as 'TIME (% (til N) 86400000))) +(set K7 (as 'TIMESTAMP (* (til N) 1000000))) +(set K8 (as 'SYM (% (til N) 5000))) +(set K9 (+ 63 (* 0 (til N)))) +(set K10 (+ 70 (* 0 (til N)))) +(set K11 (+ 77 (* 0 (til N)))) +(set K12 (+ 84 (* 0 (til N)))) +(set K13 (+ 91 (* 0 (til N)))) +(set K14 (+ 98 (* 0 (til N)))) +(set K15 (+ 105 (* 0 (til N)))) +(set K16 (+ 112 (* 0 (til N)))) +(set VV (til N)) +(set KL (list K0 K1 K2 K3 K4 K5 K6 K7 K8 K9 K10 K11 K12 K13 K14 K15 K16 VV)) +(set T17 (table [k0 k1 k2 k3 k4 k5 k6 k7 k8 k9 k10 k11 k12 k13 k14 k15 k16 v] KL)) +(set R17 (select {c: (count v) from: T17 by: {k0: k0 k1: k1 k2: k2 k3: k3 k4: k4 k5: k5 k6: k6 k7: k7 k8: k8 k9: k9 k10: k10 k11: k11 k12: k12 k13: k13 k14: k14 k15: k15 k16: k16}})) +(count R17) -- 100000 +;; k0 is distinct per row => every group is a singleton, first-seen == input order +(all (== (at R17 'k0) K0)) -- true +(all (== (at R17 'k1) K1)) -- true +(all (== (at R17 'k2) K2)) -- true +(all (== (at R17 'k3) K3)) -- true +(all (== (at R17 'k4) K4)) -- true +(all (== (at R17 'k5) K5)) -- true +(all (== (at R17 'k6) K6)) -- true +(all (== (at R17 'k7) K7)) -- true +(all (== (at R17 'k8) K8)) -- true +(all (== (at R17 'k9) K9)) -- true +(all (== (at R17 'k16) K16)) -- true +(all (== (at R17 'c) (+ 1 (* 0 (til N))))) -- true + +;; ---------- 3. take-bounded emit (serial range call) ---------- +(set RT (select {c: (count v) from: T17 by: {k0: k0 k1: k1 k2: k2 k3: k3 k4: k4 k5: k5 k6: k6 k7: k7 k8: k8 k9: k9 k10: k10 k11: k11 k12: k12 k13: k13 k14: k14 k15: k15 k16: k16} take: 10})) +(count RT) -- 10 +(all (== (at RT 'k0) (til 10))) -- true +(all (== (at RT 'k1) (as 'I32 (til 10)))) -- true +(all (== (at RT 'k3) (as 'U8 (til 10)))) -- true +(all (== (at RT 'k8) (as 'SYM (til 10)))) -- true +(all (== (at RT 'k16) (+ 112 (* 0 (til 10))))) -- true + +;; same for the wide-SYM shape: bounded emit agrees with the full emit +(set RSB (select {c: (count v) from: TS by: k take: 10})) +(count RSB) -- 10 +(all (== (at RSB 'k) (as 'SYM (til 10)))) -- true diff --git a/test/rfl/group/topn_asc_take.rfl b/test/rfl/group/topn_asc_take.rfl new file mode 100644 index 00000000..93c15509 --- /dev/null +++ b/test/rfl/group/topn_asc_take.rfl @@ -0,0 +1,78 @@ +;; ════════════════════════════════════════════════════════════════════ +;; topn_asc_take.rfl — direction of the group emit filter (issue #408). +;; +;; `select … by … asc: take: N` arms the SAME emit filter as the +;; `desc:` form (query.c match_group_desc_count_take sets .desc = 0), and +;; group.c serves it either with the bounded top-N heap (use_topn_filter +;; → topn_fuse_partition / topn_scan_fn → topn_heap_push, whose +;; TOPN_BETTER predicate is direction-symmetric) or, on single-key +;; dense/sparse shapes, with the desc-only keep-min trims. Two places used to force +;; `desc = 1` whenever the ordering agg was COUNT — the v2_emit direction +;; hoist and group_emit_filter_trim — so an asc COUNT take returned the +;; LARGEST N groups, byte-identical to the desc answer, while `-c 1` (no +;; pool ⇒ no radix emit filter) returned the correct smallest N. The +;; answer therefore changed with core count. Regression: #408. +;; +;; PATH REACH. Every fixture below is 2.5M rows (> the 1<<21 radix +;; admission) with the emit filter armed by `asc:`/`desc: take: N`, +;; N ≤ 1024. Instrumentation showed the 2-KEY selects entering the +;; parallel radix path with the fused per-partition heap armed +;; (desc_dir = 0 for `asc:`, 1 for `desc:`). The SINGLE-key selects +;; take the dense/sparse keep-min ladder instead: desc arms +;; da_count_emit_keep_min, asc must NOT (that machinery is largest- +;; first only) and falls through to the full group set + sort/take — +;; pre-fix, single-key asc was wrong at EVERY core count, not just +;; under the pool. +;; +;; ORACLE. S = round(sqrt(i)) over (til 2500000) puts 2j+1 rows in group +;; j (1, 3, 5, … — every group count DISTINCT, so the smallest-N set is +;; unambiguous and tie-break order cannot mask a direction flip). Group +;; ids are the counts' rank, so the key column is checked too: an asc +;; take:5 must return groups 0..4, a desc take:5 the last five. +;; ════════════════════════════════════════════════════════════════════ + +(set T408 (til 2500000)) +(set S408 (as 'I64 (sqrt (as 'F64 T408)))) +(set G408 (as 'SYM S408)) +;; w cycles over [-500, 512] with period 1013 (coprime with the group +;; sizes) so SUM/MIN/MAX have a genuine spread across groups. +(set W408 (- (% T408 1013) 500)) +(set B408 (table [g k v w] (list G408 S408 T408 W408))) + +;; ─── COUNT, two keys (SYM + I64) — the shape from the issue ────────── +(set A408 (select {c: (count v) by: {g: g k: k} from: B408 asc: c take: 5})) +(count A408) -- 5 +;; smallest five counts are 1, 3, 5, 7, 9 … +(at (at A408 'c) 0) -- 1 +(at (at A408 'c) 4) -- 9 +(sum (at A408 'c)) -- 25 +;; … held by groups 0, 1, 2, 3, 4 (before the fix: the five LARGEST). +(at (at A408 'k) 0) -- 0 +(sum (at A408 'k)) -- 10 + +(set D408 (select {c: (count v) by: {g: g k: k} from: B408 desc: c take: 5})) +(count D408) -- 5 +(at (at D408 'c) 0) -- 3161 +(sum (at D408 'c)) -- 15785 + +;; ─── COUNT, single I64 key (same filter, dense/sparse entry) ───────── +(set B408S (table [k v] (list S408 T408))) +(set A408S (select {c: (count v) by: {k: k} from: B408S asc: c take: 5})) +(sum (at A408S 'c)) -- 25 +(sum (at A408S 'k)) -- 10 +(sum (at (select {c: (count v) by: {k: k} from: B408S desc: c take: 5}) 'c)) -- 15785 + +;; ─── SUM: asc keeps the most negative sums, desc the largest ───────── +(at (at (select {s: (sum w) by: {g: g k: k} from: B408 asc: s take: 5}) 's) 0) -- -114612 +(at (at (select {s: (sum w) by: {g: g k: k} from: B408 desc: s take: 5}) 's) 0) -- 133674 + +;; ─── MIN / MAX: the direction is the agg's, not the op's ──────────── +;; `asc: ` asks for the groups with the SMALLEST maximum — the tiny +;; leading groups, whose w never climbs past -500 + (rows-1). Under the +;; old COUNT-only coercion these were unaffected, but they pin that the +;; hoisted direction stays per-request for every supported agg. +(at (at (select {m: (max w) by: {g: g k: k} from: B408 asc: m take: 5}) 'm) 0) -- -500 +(at (at (select {m: (max w) by: {g: g k: k} from: B408 asc: m take: 5}) 'm) 4) -- -476 +(at (at (select {m: (max w) by: {g: g k: k} from: B408 desc: m take: 5}) 'm) 0) -- 512 +;; `desc: ` — the groups whose minimum is largest. +(at (at (select {m: (min w) by: {g: g k: k} from: B408 desc: m take: 5}) 'm) 0) -- 423 diff --git a/test/rfl/integration/math.rfl b/test/rfl/integration/math.rfl index e0b668c7..155c660d 100644 --- a/test/rfl/integration/math.rfl +++ b/test/rfl/integration/math.rfl @@ -2478,3 +2478,78 @@ ;; Test division with type conversion (/ [100 200 300] 3.0) -- [33.33 66.67 100.0] (% [100 200 300] 3.0) -- [1.0 2.0 0.0] + +;; ========== I64 FLOOR DIV EXACTNESS / UB REGRESSIONS ========== +;; Integer div must stay in int64 space: the old double round-trip lost +;; precision above 2^53 and hit UB at q == 2^63 (div -9223372036854775807 -1). +(div 9007199254740993 1) -- 9007199254740993 +(div 9007199254740993 3) -- 3002399751580331 +(div 18014398509481985 1) -- 18014398509481985 +(div 123456789012345678 1) -- 123456789012345678 +(div 9223372036854775807 1) -- 9223372036854775807 +(div 9223372036854775807 -1) -- -9223372036854775807 +(div -9223372036854775807 -1) -- 9223372036854775807 +(div -9223372036854775807 2) -- -4611686018427387904 +(div -9223372036854775807 -2) -- 4611686018427387903 +(div 9223372036854775807 2) -- 4611686018427387903 +;; vectors (1-element vectors route through the scalar path) +(div [9007199254740993 9007199254740993] 3) -- [3002399751580331 3002399751580331] +(div [123456789012345678 9007199254740993] 1) -- [123456789012345678 9007199254740993] +(div [9223372036854775807] -1) -- [-9223372036854775807] +(div [-9223372036854775807] -1) -- [9223372036854775807] +;; out-of-int64-range (float input) still clamps to null +(div 1e308 1) -- 0Nl +(div -1e308 1) -- 0Nl +(div [1e308 9.3e18] 1) -- [0Nl 0Nl] +(div 9223372036854775808.0 1) -- 0Nl +;; floor semantics preserved for int operands +(div [-7 7 -7] [2 -2 -2]) -- [-4 -4 3] +(div [7 8 9] 2) -- [3 4 4] +;; compiled select/update must not round integer columns through double +(set TdivI64 (table [v] (list [9007199254740993 123456789012345678]))) +(at (select {q: (div v 1) from: TdivI64}) 'q) -- [9007199254740993 123456789012345678] +(at (select {q: (div v 3) from: TdivI64}) 'q) -- [3002399751580331 41152263004115226] +(set TdivUpd (table [v q] (list [9007199254740993 123456789012345678] [0 0]))) +(at (update {q: (div v 1) from: TdivUpd}) 'q) -- [9007199254740993 123456789012345678] +(at (update {q: (div v 3) from: TdivUpd}) 'q) -- [3002399751580331 41152263004115226] + +;; ── dual-path: the fast (<=2^53 double) and exact (>2^53 int64) kernels +;; must agree. 2^53 itself takes the fast path; 2^53+1 the exact one. ── +(div 9007199254740992 3) -- 3002399751580330 +(div [9007199254740992 9007199254740993] 1) -- [9007199254740992 9007199254740993] +;; a mixed small+big morsel forces the exact kernel for the whole span +(set TdivMix (table [a b] (list [10 9007199254740993] [3 7]))) +(at (select {q: (div a b) from: TdivMix}) 'q) -- [3 1286742750677284] +;; div-by-zero through the integer path: scalar/vector null, narrow arm 0 +(div 5 0) -- 0Nl +(div [5 6] [0 2]) -- [0Nl 3] +(set TdivZ (table [a b] (list [5 6] [0 2]))) +(at (select {q: (div a b) from: TdivZ}) 'q) -- [0Nl 3] +;; narrow-output floor div (I32 / I16 / U8 arms), incl. div-by-zero -> 0 +(div (as 'I32 [100 55 -7]) (as 'I32 [7 8 2])) -- (as 'I32 [14 6 -4]) +(div (as 'I16 [100 55 -7]) (as 'I16 [7 8 2])) -- (as 'I16 [14 6 -4]) +(set TdivN (table [a b] (list (as 'I16 [100 55 6]) (as 'I16 [7 0 2])))) +(at (select {q: (div a b) from: TdivN}) 'q) -- (as 'I16 [14 0 3]) + +;; ── #403 follow-up: the ±2^53 gate rides inside the I64 divide loop for +;; `I64 column ÷ (I64 column | int scalar)`, so the column is read once +;; instead of twice. Same all-or-nothing decision: one out-of-range value +;; (or a null, which is INT64_MIN) sends the whole range to the exact +;; kernel. These pin fast lane, redo lane, and the scalar-divisor lane. ── +(set TdivGate (table [a b] (list [10 -10 9007199254740993 0Nl] [3 3 7 3]))) +(at (select {q: (div a b) from: TdivGate}) 'q) -- [3 -4 1286742750677284 0Nl] +(at (select {q: (div a 3) from: TdivGate}) 'q) -- [3 -4 3002399751580331 0Nl] +(at (select {q: (div a 9007199254740993) from: TdivGate}) 'q) -- [0 -1 1 0Nl] +;; all-small column keeps the fast lane (identical results either way) +(set TdivGateS (table [a b] (list [10 -10 20 -21] [3 3 7 4]))) +(at (select {q: (div a b) from: TdivGateS}) 'q) -- [3 -4 2 -6] +(at (select {q: (div a 3) from: TdivGateS}) 'q) -- [3 -4 6 -7] +(at (select {q: (div a 0) from: TdivGateS}) 'q) -- [0Nl 0Nl 0Nl 0Nl] +;; broadcast scalar past ±2^53 with an in-range column: exact kernel, no rescan +(at (select {q: (div a -9007199254740993) from: TdivGateS}) 'q) -- [-1 0 -1 0] +;; gate BOUND pin: values strictly inside (2^53, 2^54], no null, nothing +;; larger — the only fixture where widening the ±2^53 bound (0x1p53 -> +;; 0x1p54) changes the answer instead of being masked by a co-resident +;; out-of-range value. Verified to fail on exactly that mutant. +(set TdivBound (table [v] (list [9007199254740993 9007199254740995 9007199254740997]))) +(at (select {q: (div v 1) from: TdivBound}) 'q) -- [9007199254740993 9007199254740995 9007199254740997] diff --git a/test/rfl/join/str_key_nyi.rfl b/test/rfl/join/str_key_nyi.rfl new file mode 100644 index 00000000..8ec45a0d --- /dev/null +++ b/test/rfl/join/str_key_nyi.rfl @@ -0,0 +1,33 @@ +;; str_key_nyi.rfl — STR equality-key columns are unsupported in the asof and +;; window join kernels and must be DECLINED, not silently mis-answered. +;; +;; Root cause: both kernels read equality-key cells through read_col_i64 +;; (asof via asof_eq_lread; window when it sorts/probes the right side), and +;; read_col_i64 has no RAY_STR case — a STR column falls into the byte-wide +;; default. asof then mismatches outright (a key that should match nulls +;; out); window only reads one byte per row, so small key sets coincide by +;; luck but enough distinct keys collide and cross-contaminate groups, so the +;; aggregates are WRONG. The base equi-join (inner/left/anti) uses a separate +;; STR-aware kernel and stays correct, so the fix is scoped to asof + window: +;; reject a STR eq-key on either side with `nyi` instead of returning corrupt +;; data. window-join1 shares the window kernel; the 4-argument compatibility +;; window forms fall through to the asof executor, so they need the same guard. + +;; ── asof-join: STR equality key → nyi (both operand orders) ────────── +(set tr (table [Sym Time Price] (list ["A" "A"] [12:00:01 12:00:04] [100 101])))(set qs (table [Sym Time Bid] (list ["A" "A"] [12:00:00 12:00:05] [99 100])))(asof-join [Sym Time] tr qs) !- nyi +(set tr (table [Sym Time Price] (list ["A" "A"] [12:00:01 12:00:04] [100 101])))(set qs (table [Sym Time Bid] (list ["A" "A"] [12:00:00 12:00:05] [99 100])))(asof-join [Sym Time] qs tr) !- nyi + +;; ── window-join / window-join1: STR equality key → nyi ─────────────── +(set wt (table [Sym Time Price] (list ["a" "a"] [10:00:01.000 10:00:05.000] [100 200])))(set wq (table [Sym Time Bid] (list ["a" "a" "a"] [10:00:00.000 10:00:02.000 10:00:04.000] [99 100 101])))(set wi (map-left + [-2000 2000] (at wt 'Time)))(window-join [Sym Time] wi wt wq {minBid: (min Bid)}) !- nyi +(set wt (table [Sym Time Price] (list ["a" "a"] [10:00:01.000 10:00:05.000] [100 200])))(set wq (table [Sym Time Bid] (list ["a" "a" "a"] [10:00:00.000 10:00:02.000 10:00:04.000] [99 100 101])))(set wi (map-left + [-2000 2000] (at wt 'Time)))(window-join1 [Sym Time] wi wt wq {minBid: (min Bid)}) !- nyi +(set wt (table [Sym Time Price] (list ["a" "a"] [10:00:01.000 10:00:05.000] [100 200])))(set wq (table [Sym Time Bid] (list ["a" "a" "a"] [10:00:00.000 10:00:02.000 10:00:04.000] [99 100 101])))(at (window-join wt wq [Sym] 'Time) 'Bid) !- nyi +(set wt (table [Sym Time Price] (list ["a" "a"] [10:00:01.000 10:00:05.000] [100 200])))(set wq (table [Sym Time Bid] (list ["a" "a" "a"] [10:00:00.000 10:00:02.000 10:00:04.000] [99 100 101])))(at (window-join1 wt wq [Sym] 'Time) 'Bid) !- nyi + +;; ── SYM equality keys keep working and stay order-independent ──────── +(set KS (as 'sym ["AAPL" "AAPL"]))(set trS (table [Sym Time Price] (list KS [12:00:01 12:00:04] [100 101])))(set qsS (table [Sym Time Bid] (list KS [12:00:00 12:00:05] [99 100])))(at (asof-join [Sym Time] trS qsS) 'Bid) -- [99 99] +(set KS (as 'sym ["AAPL" "AAPL"]))(set trS (table [Sym Time Price] (list KS [12:00:01 12:00:04] [100 101])))(set qsU (table [Sym Time Bid] (list KS [12:00:05 12:00:00] [100 99])))(at (asof-join [Sym Time] trS qsU) 'Bid) -- [99 99] +(set KW (as 'sym ["a" "a"]))(set wtS (table [Sym Time Price] (list KW [10:00:01.000 10:00:05.000] [100 200])))(set wqS (table [Sym Time Bid] (list (as 'sym ["a" "a" "a"]) [10:00:00.000 10:00:02.000 10:00:04.000] [99 100 101])))(set wiS (map-left + [-2000 2000] (at wtS 'Time)))(at (window-join [Sym Time] wiS wtS wqS {minBid: (min Bid)}) 'minBid) -- [99 100] + +;; ── base equi-join must NOT over-reject: STR keys stay correct ─────── +(set A (table [K V] (list ["x" "y"] [1 2])))(set B (table [K W] (list ["x" "z"] [10 20])))(at (inner-join [K] A B) 'V) -- [1] +(set A (table [K V] (list ["x" "y"] [1 2])))(set B (table [K W] (list ["x" "z"] [10 20])))(at (anti-join [K] A B) 'V) -- [2] diff --git a/test/rfl/query/count_distinct_fused.rfl b/test/rfl/query/count_distinct_fused.rfl new file mode 100644 index 00000000..4ee5af17 --- /dev/null +++ b/test/rfl/query/count_distinct_fused.rfl @@ -0,0 +1,41 @@ +;; Fused grouped count-distinct (spec Part B). Closed-form data: +;; 300K rows, k = i mod 1000 (1000 groups), v = i mod 3000. +;; For each k the values are {k, k+1000, k+2000} mod 3000 pattern: +;; v = i mod 3000 and k = i mod 1000 => v mod 1000 == k, and v takes +;; exactly 3 distinct values per k. So every group's distinct count is 3. +(set CD (table [k v] (list (% (til 300000) 1000) (% (til 300000) 3000)))) +(count (select {u: (count (distinct v)) by: k from: CD})) -- 1000 +(sum (at (select {u: (count (distinct v)) by: k from: CD}) 'u)) -- 3000 +(at (at (select {u: (count (distinct v)) by: k from: CD}) 'k) 0) -- 0 +;; duplicate-heavy: v constant => distinct count 1 per group +(set CD1 (table [k v] (list (% (til 300000) 1000) (* (til 300000) 0)))) +(sum (at (select {u: (count (distinct v)) by: k from: CD1}) 'u)) -- 1000 +;; top-N by distinct count still works through the emit path +(count (select {u: (count (distinct v)) by: k from: CD desc: u take: 7})) -- 7 + +;; SYM-key coverage (finding I2): ray_cd_fused's cdf_type_ok accepts SYM +;; keys directly (src/ops/cdfuse.c), so exercise the fused kernel with a +;; SYM group key at the same 300K-row / 1000-key scale as CD above, built +;; via (as 'SYM (% (til N) K)) — the same construction test/rfl/group's +;; group_radix_coverage.rfl and parted_group_expr_key.rfl use for large +;; synthetic SYM columns. v = (til N), so every row has a unique value: +;; each of the 1000 keys owns exactly 300 rows, all distinct => u == 300 +;; per group, closed-form sum == N. +(set CDS (table [k v] (list (as 'SYM (% (til 300000) 1000)) (til 300000)))) +(count (select {u: (count (distinct v)) by: k from: CDS})) -- 1000 +(sum (at (select {u: (count (distinct v)) by: k from: CDS}) 'u)) -- 300000 +(at (at (select {u: (count (distinct v)) by: k from: CDS}) 'u) 0) -- 300 + +;; Boundary pin (finding I3): CDF_MIN_ROWS is 262144 (src/ops/cdfuse.h), so +;; nrows==262143 takes the two-pass group+count-distinct rewrite while +;; nrows==262144 takes ray_cd_fused. k = i mod 1000, v = i mod 3: since +;; gcd(1000,3)==1 and every key has >=262 occurrences (>=3), each group's v +;; values cycle through all 3 residues, so u==3 for every one of the 1000 +;; keys regardless of which side of the boundary the row count falls on — +;; both paths must agree on count==1000 and sum(u)==3000. +(set CDB0 (table [k v] (list (% (til 262143) 1000) (% (til 262143) 3)))) +(count (select {u: (count (distinct v)) by: k from: CDB0})) -- 1000 +(sum (at (select {u: (count (distinct v)) by: k from: CDB0}) 'u)) -- 3000 +(set CDB1 (table [k v] (list (% (til 262144) 1000) (% (til 262144) 3)))) +(count (select {u: (count (distinct v)) by: k from: CDB1})) -- 1000 +(sum (at (select {u: (count (distinct v)) by: k from: CDB1}) 'u)) -- 3000 diff --git a/test/rfl/query/group_f64_signed_zero.rfl b/test/rfl/query/group_f64_signed_zero.rfl new file mode 100644 index 00000000..235a6eb1 --- /dev/null +++ b/test/rfl/query/group_f64_signed_zero.rfl @@ -0,0 +1,105 @@ +;; F64 group keys: -0.0 and +0.0 are ONE group (issue #407). +;; +;; ray_hash_f64 normalises -0.0 -> +0.0, but group_keys_equal / ght_lanes_equal +;; compare the raw 8 key bytes. A column carrying both bit patterns therefore +;; hashed -0.0 into +0.0's slot chain but never compared equal to it, so -0.0 +;; split off into its own group. Every F64 group-key builder now canonicalises +;; the value at the key-READ boundary (group_key_f64_bits in src/ops/group.c) before +;; both the key store and the hash, so hash and compare see identical bits on +;; every path -- generic per-row, fat-entry radix phase 1, the v2 partition- +;; major stager, the reprobe/regroup paths and the sequential fallback. +;; +;; EVERY -0.0 below is produced at RUNTIME via (* -1.0 0.0). A -0.0 LITERAL is +;; not enough: the release build compiles with -fno-signed-zeros and folds the +;; literal to +0.0 before the group kernel ever sees it, which is exactly why +;; the bug survived in release. The multiply is data-dependent, so no build +;; folds it away. (The literal form is pinned in group_packed_key.rfl, which +;; covers the debug side of the same contract.) +;; +;; NaN policy: NaN is deliberately NOT canonicalised. Distinct NaN payloads +;; hash distinctly AND compare distinctly -- self-consistent already, unlike +;; the -0.0 asymmetry -- and SQL nullness rides on the key null mask, not on +;; the payload, so 0Nf keeps its own group (asserted below). + +(set Z (* -1.0 0.0)) + +;; ── single F64 key, runtime -0.0 ───────────────────────────────────────── +(set SZ1 (table [f v] (list (as 'F64 (list 0.0 Z 1.0)) [1 2 3]))) +(count (select {n: (count v) by: {f: f} from: SZ1})) -- 2 +(at (select {n: (count v) by: {f: f} asc: f from: SZ1}) 'n) -- [2 1] + +;; -0.0 arriving FIRST must not change the answer (the group's stored key is +;; canonical +0.0 for every contributing row, so the representative does not +;; depend on which row opened the group or on which worker saw it first). +(set SZ2 (table [f v] (list (as 'F64 (list Z 0.0 1.0)) [1 2 3]))) +(count (select {n: (count v) by: {f: f} from: SZ2})) -- 2 +(at (select {n: (count v) by: {f: f} asc: f from: SZ2}) 'n) -- [2 1] + +;; a column of nothing but runtime -0.0 collapses to a single group +(count (select {n: (count v) by: {f: f} from: (table [f v] (list (as 'F64 (list Z Z Z)) [1 2 3]))})) -- 1 + +;; group count must not depend on how often the rows repeat +(- (count (select {n: (count v) by: {f: f} from: SZ1})) (count (select {n: (count v) by: {f: f} from: (table [f v] (list (as 'F64 (list 0.0 Z 1.0 0.0 Z 1.0)) [1 2 3 4 5 6]))}))) -- 0 + +;; ── multi-key: F64 + F64, and F64 + I64 ────────────────────────────────── +(set SZM (table [a b v] (list (as 'F64 (list 0.0 Z 1.0 Z 0.0)) (as 'F64 (list 1.0 1.0 2.0 2.0 1.0)) [1 2 3 4 5]))) +(count (select {n: (count v) by: {a: a b: b} from: SZM})) -- 3 +(at (select {n: (count v) by: {a: a b: b} from: SZM}) 'n) -- [3 1 1] + +(set SZI (table [a k v] (list (as 'F64 (list 0.0 Z 0.0 Z 1.0)) (as 'I64 (list 7 7 8 8 7)) [1 2 3 4 5]))) +(count (select {n: (count v) by: {a: a k: k} from: SZI})) -- 3 +(at (select {n: (count v) by: {a: a k: k} from: SZI}) 'n) -- [2 2 1] + +;; ── WHERE-filtered group-by (rowsel / match-list key gather) ───────────── +;; Rows 0..3 survive: +0.0, -0.0, 1.0, -0.0 -> 2 groups, counts 3 and 1. +(set SZW (table [f k v] (list (as 'F64 (list 0.0 Z 1.0 Z 0.0 Z)) (as 'I64 (list 0 1 2 3 9 9)) [1 2 3 4 5 6]))) +(count (select {n: (count v) by: {f: f} from: SZW where: (< k 4)})) -- 2 +(at (select {n: (count v) by: {f: f} asc: f from: SZW where: (< k 4)}) 'n) -- [3 1] +(sum (at (select {s: (sum v) by: {f: f} from: SZW where: (< k 4)}) 's)) -- 10 + +;; ── 0Nf null F64 keys keep their own group (NaN is not canonicalised) ──── +;; p = [+0.0, null, -0.0, null, 1.0, +0.0] -> 3 groups: 0.0 x3, null x2, 1.0 x1. +;; Base behaviour for the NULL group is unchanged; only the +0.0 / -0.0 split +;; is gone (base produced 4 groups here). +(set SZN (table [i p] (list (til 6) (as 'F64 (list 0.0 0Nf Z 0Nf 1.0 0.0))))) +(count (select {n: (count i) by: {p: p} from: SZN})) -- 3 +(at (select {n: (count i) by: {p: p} from: SZN}) 'n) -- [3 2 1] +(nil? (at (at (select {n: (count i) by: {p: p} from: SZN}) 'p) 1)) -- true +(nil? (at (at (select {n: (count i) by: {p: p} from: SZN}) 'p) 0)) -- false +;; a null-only F64 key column is still one group, as before +(count (select {n: (count i) by: {p: p} from: (table [i p] (list (til 3) (as 'F64 (list 0Nf 0Nf 0Nf))))})) -- 1 + +;; ── large parallel: 3M rows > the radix pipeline's 1<<21 scan gate ─────── +;; Every expectation below is analytic and core-count independent, so a +;; divergence between the sequential and the pool-parallel builders (phase-1 +;; stagers, phase-2 partition merge, reprobe) fails loudly. +(set BN 3000000) + +;; low cardinality: f cycles [+0.0, -0.0, 1.0, -1.0] -> 3 groups, the merged +;; zero group holding exactly half the rows. +(set BF (as 'F64 (take (list 0.0 Z 1.0 -1.0) BN))) +(set BT (table [f k v] (list BF (% (til BN) 997) (til BN)))) +(count (select {n: (count v) by: {f: f} from: BT})) -- 3 +(at (select {n: (count v) by: {f: f} asc: f from: BT}) 'n) -- [750000 1500000 750000] +(sum (at (select {n: (count v) by: {f: f} from: BT}) 'n)) -- 3000000 + +;; two keys (F64 + I64), 3 x 997 combinations -- all reachable because 4 and +;; 997 are coprime and BN covers every residue mod 3988. +(count (select {n: (count v) by: {f: f k: k} from: BT})) -- 2991 +(sum (at (select {n: (count v) by: {f: f k: k} from: BT}) 'n)) -- 3000000 + +;; large + WHERE: k < 100 of 997 survives; both zero signs are in the survivors. +(count (select {n: (count v) by: {f: f} from: BT where: (< k 100)})) -- 3 +(at (select {n: (count v) by: {f: f} asc: f from: BT where: (< k 100)}) 'n) -- [75232 150463 75232] +(sum (at (select {n: (count v) by: {f: f} from: BT where: (< k 100)}) 'n)) -- 300927 + +;; medium-high cardinality F64 key (~200K groups) so the near-unique +;; partition-major probe runs: magnitude i%99991 with a parity-driven sign, so +;; +m and -m both occur for every m -- and m == 0 yields both +0.0 and -0.0, +;; which must fold into ONE group. 2*99990 + 1 = 199981. +(set BU (* (- (* 2 (% (til BN) 2)) 1) (as 'F64 (% (til BN) 99991)))) +(count (select {n: (count v) by: {f: f} from: (table [f v] (list BU (til BN)))})) -- 199981 +;; distinct uses the hashset path, whose F64 compare is `==` (so it always +;; folded the zeros) -- an independent oracle for the same count. +(count (distinct BU)) -- 199981 +(sum (at (select {n: (count v) by: {f: f} from: (table [f v] (list BU (til BN)))}) 'n)) -- 3000000 diff --git a/test/rfl/query/group_negative_narrow_keys.rfl b/test/rfl/query/group_negative_narrow_keys.rfl new file mode 100644 index 00000000..597c60d8 --- /dev/null +++ b/test/rfl/query/group_negative_narrow_keys.rfl @@ -0,0 +1,23 @@ +;; Direct-array group-by: signed narrow keys (I16/I32/DATE/TIME) must be +;; sign-extended identically in the min/max prescan and the accumulate loop. +;; The accumulate dispatch used to pick readers by ELEMENT SIZE only, reading +;; I16 as uint16 — a negative key then mapped to a slot far outside the dense +;; range (out-of-bounds write) and its group vanished from the result +;; (caught by csv_explicit_numeric_types.rfl's 3-group check). + +;; I16 key with a negative group, binary agg (wavg keeps DA eligible) +(set Tnk (table [k a b c] (list [32000 -300 9 9] [1.25 -2.5 3.75 16777216.0] [100000 -7 42 100001] [0.25 1.5 -0.25 1.0]))) +(count (select {mx: (max a) wa: (wavg b c) by: {kk: k} from: Tnk})) -- 3 +(at (at (select {mx: (max a) wa: (wavg b c) by: {kk: k} asc: kk from: Tnk}) 'kk) 0) -- -300 + +;; I32-typed negative keys via wide values +(count (select {c: (count v) by: {k: k} from: (table [k v] (list [-100000 -100000 70000 3] [1 2 3 4]))})) -- 3 + +;; sums attributed to the right (negative) group +(at (at (select {s: (sum v) by: {k: k} where: (< k 0) from: (table [k v] (list [-5 -5 3 7] [10 20 30 40]))}) 's) 0) -- 30 + +;; two narrow signed keys (composite gid path) +(count (select {c: (count a) by: {k1: k1 k2: k2} from: (table [k1 k2 a] (list [-300 -300 9 9] [1 1 1 2] [10 20 30 40]))})) -- 3 + +;; mixed signedness composite: signed I16 + unsigned-style small ints +(count (select {c: (count a) by: {k1: k1 k2: k2} from: (table [k1 k2 a] (list [-2 -2 2 2] [0 1 0 1] [1 2 3 4]))})) -- 4 diff --git a/test/rfl/query/group_packed_key.rfl b/test/rfl/query/group_packed_key.rfl new file mode 100644 index 00000000..936557a9 --- /dev/null +++ b/test/rfl/query/group_packed_key.rfl @@ -0,0 +1,182 @@ +;; Packed-key group-by pipeline (src/ops/group.c ght_layout_t.packed_key). +;; +;; When every group key is a plain fixed-width integer lane (whitelisted type, +;; no GUID/STR indirection, no F64) the group pipeline hashes the WHOLE key +;; region in one ght_hash_lanes avalanche instead of hashing each key and +;; combining, elides the null-mask words when no key column can be null, and +;; drops the dead value slot reserved for OP_COUNT. Group ids, first-seen +;; ordering and emit-filter behaviour must be unchanged. +;; +;; There is NO 16-byte / 2-lane cap: eligibility is per-key TYPE, not tuple +;; width, so a 9-key tuple is packed just as a 2-key one is. The only +;; width-sensitive thing is P1_MORSEL_KEYS (8), which selects the morsel +;; stager vs the row-major builder — both packed. +;; +;; PATH REACH — the point of the 2.5M sizes below. The fat-entry radix +;; pipeline (radix_phase1_fn, which owns the morsel stager) needs +;; n_scan > 1<<21 AND the emit filter armed (`desc: take: N`) AND a key +;; shape the fused count-only radix declines (a SYM key does that); without a +;; SYM key the same size lands on radix_v2_phase1_fn's partition-major morsel +;; instead. Instrumenting both showed the cases below execute, in order: +;; radix_phase1_fn morsel stager nk=2 nv=0 (BIGS) +;; radix_phase1_fn morsel stager nk=3 nv=2 (BIGM) +;; radix_phase1_fn row-major packed nullable, null_words=1 (BIGN) +;; radix_v2_phase1_fn morsel packed=1 (BIGP) packed=0 (PKF) +;; radix_v2_phase1_fn morsel need_flags=SUM, near-unique (BIGU) +;; The 300K cases at the bottom cover the sequential / direct-array entries +;; into the same layout. +;; +;; Columns are derived from one shared (til 2500000) — rebuilding it per +;; column roughly doubled this file's debug-build runtime. + +(set T (til 2500000)) +(set G (as 'SYM (% T 900))) +(set K (% T 700)) +(set M (as 'I16 (- (% T 60) 30))) + +;; ── 2-key packed tuple, count-only: no value slots at all ──────────────── +;; SYM + I64 keys -> fat-entry radix + morsel stager (nk=2, nv=0). +(set BIGS (table [g k v] (list G K T))) +(count (select {c: (count v) by: {g: g k: k} from: BIGS})) -- 6300 +(sum (at (select {c: (count v) by: {g: g k: k} from: BIGS}) 'c)) -- 2500000 +(at (at (select {c: (count v) by: {g: g k: k} from: BIGS desc: c take: 10}) 'c) 0) -- 397 +;; same shape under a WHERE. The predicate is deliberately WIDE (96% of rows +;; survive) and the emit filter is kept armed: a dense selection stays on the +;; sequential scan with a rowsel, which is what drives the morsel stager's +;; INDEXED gather (group_keys_gather) instead of the contiguous one — verified +;; as contig=0. A narrow predicate, or dropping `desc/take`, routes the query +;; off this path entirely. +(count (select {c: (count v) by: {g: g k: k} from: BIGS where: (< v 2400000) desc: c take: 5})) -- 5 +(sum (at (select {c: (count v) by: {g: g k: k} from: BIGS where: (< v 2400000) desc: c take: 5}) 'c)) -- 1905 + +;; ── 2-key packed tuple with NO SYM key ─────────────────────────────────── +;; The fused count-only radix accepts this shape, so it lands on +;; radix_v2_phase1_fn's partition-major morsel path. That path was +;; UNREACHABLE until null-mask elision made its `null_words == 0` gate +;; satisfiable, and it staged a per-key hash while hash_keys_inline — used by +;; group_ht_rehash / rebuild_slots / merge_row — returns ght_hash_lanes for +;; the same layout. After a worker-HT rehash the probe then missed and +;; inserted DUPLICATE group rows; phase 2's merge folded them, so the answers +;; stayed right and a result diff could never have caught it. Regression +;; case for that. +(set BIGP (table [a b] (list (% T 5000) (% T 3000)))) +(count (select {c: (count a) by: {a: a b: b} from: BIGP desc: c take: 10})) -- 10 +(count (select {c: (count a) by: {a: a b: b} from: BIGP})) -- 15000 +(sum (at (select {c: (count a) by: {a: a b: b} from: BIGP}) 'c)) -- 2500000 + +;; ── 2-key packed tuple, NEAR-UNIQUE, count + sum + avg ─────────────────── +;; radix_v2_phase1_fn's partition-major morsel path used to be gated on +;; `need_flags == 0` (count-only). It now also serves COUNT/SUM/AVG, which +;; are the only aggs the v2 pipeline admits at all, by reading the agg inputs +;; at probe time from the staged source row. Verified reached: nk=2, +;; need_flags=GHT_NEED_SUM, entry_stride=48. +;; +;; The reorder is partition-major, but the counting sort is STABLE and a group +;; lives entirely in one partition, so rows of a group are still visited in +;; source order — an f64 SUM accumulates in exactly the old sequence. +;; +;; Near-unique on purpose (2499990 groups from 2500000 rows): every probe +;; misses and inserts, so the worker HTs outgrow their first allocation. That +;; is also the only shape that exercises group_ht_t.grow_cap — verified to +;; jump ht_cap 256 -> 1024 in one step here instead of climbing 256->512->1024 +;; and re-hashing every live group on each rung. +;; +;; `a` collides only for the ten values 0..9 (rows T and T+2499990, which share +;; b since 2499990 is even), so `desc: c take: 10` selects exactly those ten +;; groups. Their ORDER within the tie block is not defined by the engine, so +;; every take-10 assertion below is an order-independent aggregate of the ten. +(set BIGU (table [a b v] (list (% T 2499990) (% T 2) T))) +(count (select {c: (count v) s: (sum v) av: (avg v) by: {a: a b: b} from: BIGU})) -- 2499990 +(sum (at (select {c: (count v) s: (sum v) av: (avg v) by: {a: a b: b} from: BIGU}) 'c)) -- 2500000 +(sum (at (select {c: (count v) s: (sum v) av: (avg v) by: {a: a b: b} from: BIGU}) 's)) -- 3124998750000 +(count (select {c: (count v) s: (sum v) av: (avg v) by: {a: a b: b} from: BIGU desc: c take: 10})) -- 10 +(sum (at (select {c: (count v) s: (sum v) av: (avg v) by: {a: a b: b} from: BIGU desc: c take: 10}) 'c)) -- 20 +(sum (at (select {c: (count v) s: (sum v) av: (avg v) by: {a: a b: b} from: BIGU desc: c take: 10}) 'a)) -- 45 +(sum (at (select {c: (count v) s: (sum v) av: (avg v) by: {a: a b: b} from: BIGU desc: c take: 10}) 's)) -- 24999990 + +;; ── 3-key mixed-width tuple + aggs around the valueless OP_COUNT ───────── +;; I64 + SYM + I16(negative) keys. COUNT reserves no value slot, so sum/max +;; must still land on accumulator slots 0 and 1 — asserted with the count +;; both before and after them in the projection. +(set BIGM (table [a b m v] (list K G M T))) +(count (select {c: (count v) s: (sum v) mx: (max v) by: {a: a b: b m: m} from: BIGM desc: c take: 7})) -- 7 +(count (select {c: (count v) s: (sum v) mx: (max v) by: {a: a b: b m: m} from: BIGM})) -- 6300 +(sum (at (select {s: (sum v) c: (count v) mn: (min v) by: {a: a b: b m: m} from: BIGM}) 's)) -- 3124998750000 +(max (at (select {s: (sum v) c: (count v) mx: (max v) by: {a: a b: b m: m} from: BIGM}) 'mx)) -- 2499999 +;; negative narrow key survives the packed hash and compare at scale +(at (at (select {c: (count v) by: {m: m} asc: m from: BIGM}) 'm) 0) -- -30 + +;; ── nullable keys keep the null-mask words (elision must NOT fire) ─────── +;; Lands on the packed ROW-MAJOR builder, not the morsel stager (which +;; requires null_words == 0), so both packed builders are covered. Indexing +;; a small nullable seed is what makes the column really carry HAS_NULLS. +(set NSEED (as 'I64 (list 1 0 0Ni 7 0Ni 3))) +(set BIGN (table [k j v] (list (at NSEED (% T 6)) (as 'SYM (% T 400)) T))) +(count (select {c: (count v) by: {k: k j: j} from: BIGN desc: c take: 5})) -- 5 +(count (select {c: (count v) by: {k: k j: j} from: BIGN})) -- 1000 +(sum (at (select {c: (count v) by: {k: k j: j} from: BIGN}) 'c)) -- 2500000 + +;; ── Tuple width: 8 keys vs 9 keys ──────────────────────────────────────── +;; Result-level only: at 300K rows these stay below the radix-layout cutoff, +;; so this pins values for wide key tuples but not which builder runs them +;; (the P1_MORSEL_KEYS stack-budget boundary has no path-level test). +(set K9 (table [a b c d e f g h i v] (list (% (til 300000) 2) (% (til 300000) 3) (% (til 300000) 5) (% (til 300000) 7) (% (til 300000) 11) (as 'I16 (% (til 300000) 13)) (as 'I32 (% (til 300000) 4)) (% (til 300000) 6) (as 'SYM (% (til 300000) 9)) (til 300000)))) +(count (select {n: (count v) by: {a: a b: b c: c d: d e: e f: f g: g h: h} from: K9})) -- 60060 +(count (select {n: (count v) by: {a: a b: b c: c d: d e: e f: f g: g h: h i: i} from: K9})) -- 180180 +(sum (at (select {n: (count v) by: {a: a b: b c: c d: d e: e f: f g: g h: h i: i} from: K9}) 'n)) -- 300000 + +;; ── Smaller shapes: sequential / direct-array entries, same layout ─────── +(set PK2 (table [a b] (list (% (til 300000) 500) (% (til 300000) 300)))) +(count (select {c: (count a) by: {a: a b: b} from: PK2})) -- 1500 +(sum (at (select {c: (count a) by: {a: a b: b} from: PK2}) 'c)) -- 300000 +(count (select {c: (count a) by: {a: a b: b} from: PK2 desc: c take: 10})) -- 10 +(at (at (select {c: (count a) by: {a: a b: b} from: PK2 desc: c take: 10}) 'c) 0) -- 200 +(count (select {c: (count a) by: {a: a b: b} from: PK2 where: (< a 10)})) -- 30 + +(set PKW (table [s i l v] (list (as 'I16 (- (% (til 300000) 400) 200)) (as 'I32 (- (% (til 300000) 9) 4)) (- (% (til 300000) 6) 3) (til 300000)))) +(count (select {n: (count v) by: {s: s i: i l: l} from: PKW})) -- 3600 +(sum (at (select {n: (count v) by: {s: s i: i l: l} from: PKW}) 'n)) -- 300000 +(at (at (select {n: (count v) by: {s: s} asc: s from: PKW}) 's) 0) -- -200 +(count (select {n: (count v) by: {s: s i: i} from: PKW where: (< s 0)})) -- 1800 +(sum (at (select {n: (count v) by: {i: i l: l} from: PKW where: (< v 1000)}) 'n)) -- 1000 + +;; small nullable: a NULL key stays a distinct group from a 0 key +(set PKN (table [k v] (list (as 'I64 (list 1 0 0Ni 1 0Ni 0)) [10 20 30 40 50 60]))) +(count (select {n: (count v) by: {k: k} from: PKN})) -- 3 +(sum (at (select {n: (count v) by: {k: k} from: PKN}) 'n)) -- 6 +(count (select {n: (count v) by: {k: k} from: PKN desc: n take: 2})) -- 2 +(count (select {n: (count v) by: {a: a b: b} from: (table [a b v] (list (as 'I64 (list 1 1 0Ni 0Ni 0)) (as 'I64 (list 2 0Ni 2 0Ni 0)) [1 2 3 4 5]))})) -- 5 + +;; ── NOT packed: F64 key keeps the per-key hash ─────────────────────────── +;; ±0.0 CONTRACT (#407, previously build-dependent). [0.0, -0.0, 1.0] is +;; TWO groups in EVERY build. Each F64 group-key builder canonicalises +;; -0.0 → +0.0 at the key-READ boundary (group_key_f64_bits in src/ops/group.c) +;; before both the key store and the hash, so group_keys_equal's raw-bit +;; compare can no longer disagree with ray_hash_f64's normalisation. Before +;; the fix this line was 3 in the debug/ASan build and 2 in release — release +;; only by accident, because -fno-signed-zeros folds the -0.0 LITERAL below to +;; +0.0 before the kernel sees it. The literal form is therefore the DEBUG +;; half of the contract; the release half needs a runtime-produced -0.0 and +;; lives in group_f64_signed_zero.rfl. F64 keys stay off packed_key +;; regardless (read_col_i64 does not decode F64), pinned by +;; group_extra/ght_packed_key_excludes_f64. +(count (select {n: (count v) by: {f: f} from: (table [f v] (list [0.0 -0.0 1.0] [1 2 3]))})) -- 2 +;; …and the count still does not depend on how often the rows repeat. +(- (count (select {n: (count v) by: {f: f} from: (table [f v] (list [0.0 -0.0 1.0] [1 2 3]))})) (count (select {n: (count v) by: {f: f} from: (table [f v] (list [0.0 -0.0 1.0 0.0 -0.0 1.0] [1 2 3 4 5 6]))}))) -- 0 +(set PKF (table [f k v] (list (% (til 300000) 37.0) (% (til 300000) 4) (til 300000)))) +(count (select {n: (count v) by: {f: f k: k} from: PKF})) -- 148 +(sum (at (select {n: (count v) by: {f: f k: k} from: PKF}) 'n)) -- 300000 +(count (select {n: (count v) by: {f: f k: k} from: PKF desc: n take: 6})) -- 6 + +;; ── NOT packed: STR key stays on the inline-STR / wide-key machinery ───── +(set PKT (table [s k v] (list (as 'STR (% (til 300000) 53)) (% (til 300000) 6) (til 300000)))) +(count (select {n: (count v) by: {s: s k: k} from: PKT})) -- 318 +(sum (at (select {n: (count v) by: {s: s k: k} from: PKT}) 'n)) -- 300000 + +;; ── COUNT next to value-bearing aggs at small scale too ───────────────── +(set PKA (table [k v w] (list (% (til 300000) 100) (til 300000) (% (til 300000) 3)))) +(at (at (select {n: (count v) s: (sum w) mn: (min v) mx: (max v) by: {k: k} asc: k from: PKA}) 'n) 0) -- 3000 +(at (at (select {n: (count v) s: (sum w) mn: (min v) mx: (max v) by: {k: k} asc: k from: PKA}) 'mn) 0) -- 0 +(at (at (select {n: (count v) s: (sum w) mn: (min v) mx: (max v) by: {k: k} asc: k from: PKA}) 'mx) 0) -- 299900 +(sum (at (select {n: (count v) s: (sum w) by: {k: k} from: PKA}) 's)) -- 300000 +(at (at (select {s: (sum w) n: (count v) mx: (max v) by: {k: k} asc: k from: PKA}) 'mx) 99) -- 299999 diff --git a/test/rfl/query/highcard_group.rfl b/test/rfl/query/highcard_group.rfl new file mode 100644 index 00000000..c44489ae --- /dev/null +++ b/test/rfl/query/highcard_group.rfl @@ -0,0 +1,102 @@ +;; High-cardinality group-by pinning tests for the partition-major build +;; (docs/superpowers/specs/2026-08-15-high-card-group-scaling.md, Part A). +;; 300K rows forces the parallel radix_v2 path even under RAYFORCE_CORES=2; +;; keys are arithmetic so every expected value is closed-form. + +;; ~100K distinct I64 keys, 3 rows each: count per group is exactly 3. +(set HK (table [k] (list (% (til 300000) 100000)))) +(count (select {c: (count k) by: k from: HK})) -- 100000 +(sum (at (select {c: (count k) by: k from: HK}) 'c)) -- 300000 +;; stable first-seen order: first group is key 0, last is key 99999 +(at (at (select {c: (count k) by: k from: HK}) 'k) 0) -- 0 +(at (at (select {c: (count k) by: k from: HK}) 'k) 99999) -- 99999 + +;; top-N by count with ties: every count == 3, keep semantics unchanged +(count (select {c: (count k) by: k from: HK desc: c take: 10})) -- 10 + +;; 2-key composite, 1500 distinct pairs (lcm(500,300)=1500) +(set HK2 (table [a b] (list (% (til 300000) 500) (% (til 300000) 300)))) +;; 1500 distinct pairs -> distinct pairs = 1500; count per pair = 200 +(count (select {c: (count a) by: {a: a b: b} from: HK2})) -- 1500 +(sum (at (select {c: (count a) by: {a: a b: b} from: HK2}) 'c)) -- 300000 +(at (at (select {c: (count a) by: {a: a b: b} from: HK2 desc: c take: 3}) 'c) 0) -- 200 + +;; filtered (rowsel) variant stays correct +(sum (at (select {c: (count k) by: k from: HK where: (< k 1000)}) 'c)) -- 3000 +(count (select {c: (count k) by: k from: HK where: (< k 1000)})) -- 1000 + +;; I32-typed keys through the same path +(set HK3 (table [k v] (list (as 'I32 (% (til 262144) 65536)) (til 262144)))) +(count (select {c: (count v) by: k from: HK3})) -- 65536 +(sum (at (select {c: (count v) by: k from: HK3}) 'c)) -- 262144 + +;; take-without-sort under a group-by: the take is pushed as HEAD(GROUP), so the +;; v2 radix engine emits ONLY the first N groups (bounded emit) instead of +;; materializing all 100K. The result must still be the first-seen PREFIX: +;; 100K distinct (k, j) pairs (j is a function of k), 3 rows each, so take: 5 +;; is exactly keys 0..4 with count 3. +(set HKT (table [k j] (list (% (til 300000) 100000) (% (% (til 300000) 100000) 7)))) +(set RT (select {c: (count k) by: {k: k j: j} from: HKT take: 5})) +(count RT) -- 5 +(at RT 'k) -- [0 1 2 3 4] +(at RT 'j) -- [0 1 2 3 4] +(at RT 'c) -- [3 3 3 3 3] +;; negative (tail) and range takes are NOT pushed — full group set, then slice +(at (select {c: (count k) by: {k: k j: j} from: HKT take: -3}) 'k) -- [99997 99998 99999] +(at (select {c: (count k) by: {k: k j: j} from: HKT take: [2 3]}) 'k) -- [2 3 4] + +;; ─── take-pushdown guards (grouped take: without asc:/desc:) ────────── +;; The pushdown adds HEAD(GROUP) to the DAG so the group engine can emit only +;; the first N groups. These shapes must NOT be pushed (or must survive the +;; trim), and all of them are pinned against the pre-pushdown answers. + +;; LIST-producing aggregate (top/bot): the group emits a LIST column. +(set LT (table [k v] (list (% (til 40) 6) (til 40)))) +(set LR (select {t: (top v 3) c: (count v) by: k from: LT take: 4})) +(count LR) -- 4 +(at LR 'k) -- [0 1 2 3] +(at LR 'c) -- [7 7 7 7] +(at (at LR 't) 0) -- [36 30 24] +(at (at LR 't) 3) -- [39 33 27] +;; take larger than the group count, and a negative (tail) take +(count (select {t: (top v 3) c: (count v) by: k from: LT take: 100})) -- 6 +(at (select {t: (top v 3) c: (count v) by: k from: LT take: -2}) 'k) -- [4 5] + +;; HEAD/TAIL over a table with a LIST column (exec.c trim: LIST cells are +;; copied WITH a retain — the raw byte copy used to leave the column NULL and +;; fail with "table add_col: column must be a vector"). +(set LC (table [a b] (list (as 'I64 [1 2 3]) (list [10 11] [20 21] [30 31])))) +(at (select {a: a b: b from: LC take: 2}) 'a) -- [1 2] +(at (at (select {a: a b: b from: LC take: 2}) 'b) 1) -- [20 21] +(at (select {a: a b: b from: LC take: -2}) 'a) -- [2 3] +(at (at (select {a: a b: b from: LC take: -2}) 'b) 0) -- [20 21] + +;; take: is evaluated EXACTLY once (the pushdown decision must not re-run a +;; side-effecting take: expression that apply_sort_take also reads). +(set TKCNT 0) +(count (select {c: (count v) by: k from: LT take: (do (set TKCNT (+ TKCNT 1)) 3)})) -- 3 +TKCNT -- 1 + +;; PARTED input with an EMPTY partition: exec_group_parted reads a positive +;; group_limit as "stop after N partitions", which assumes 1 group per +;; partition — partition 2 is empty, so partitions [1 2] hold only ONE group +;; and a pushed take: 2 would answer [1] instead of [1 3]. Grouped takes over +;; parted inputs are therefore never pushed. The fixture uses 1000 rows per +;; partition on purpose: exec_group_parted's cardinality gate keeps SMALL +;; partitions on the concat fallback, which never consults group_limit, so a +;; tiny fixture would pass even with the bug present. +(.sys.exec "rm -rf /tmp/rfl_take_parted") +(set TP1 (table [k v] (list (% (til 1000) 10) (til 1000)))) +(set TP0 (table [k v] (list (as 'I64 []) (as 'I64 [])))) +(.db.splayed.set "/tmp/rfl_take_parted/1/t/" TP1) +(.db.splayed.set "/tmp/rfl_take_parted/2/t/" TP0) +(.db.splayed.set "/tmp/rfl_take_parted/3/t/" TP1) +(.db.splayed.set "/tmp/rfl_take_parted/4/t/" TP1) +(.db.splayed.set "/tmp/rfl_take_parted/5/t/" TP1) +(set PTK (.db.parted.get "/tmp/rfl_take_parted/" 't)) +(at (select {c: (count v) by: part from: PTK}) 'part) -- [1 3 4 5] +(at (select {c: (count v) by: part from: PTK}) 'c) -- [1000 1000 1000 1000] +(at (select {c: (count v) by: part from: PTK take: 2}) 'part) -- [1 3] +(at (select {c: (count v) by: part from: PTK take: 2}) 'c) -- [1000 1000] +(at (select {c: (count v) by: part from: PTK take: 3}) 'part) -- [1 3 4] +(at (select {c: (count v) by: part from: PTK take: 4}) 'part) -- [1 3 4 5] diff --git a/test/rfl/query/query_update_coverage.rfl b/test/rfl/query/query_update_coverage.rfl index 3d28ee8a..cb3dc397 100644 --- a/test/rfl/query/query_update_coverage.rfl +++ b/test/rfl/query/query_update_coverage.rfl @@ -386,23 +386,24 @@ (at (window-join [Sym Time] wji_null wjt_null wjq_i64null {l: (last Price)}) 'l) -- [400] ;; ──────────────────────────────────────────────────────────────────── -;; update by: with vector-returning expression (line 8376): -;; When agg_result from exec (sub-table expression) is a vector (not atom), -;; ray_is_vec(agg_result) = true at line 8376. -;; The sub-table expression (* v 2) on each group returns a vector, -;; so line 8376 fires for the first group's result. -;; NOTE: the by-group vector result path has no broadcast logic (only -;; atoms are broadcast at 8387-8388), so new_v fills with zeros. -;; This is observable behavior, not an error. +;; update by: with a per-row VECTOR expression (`update {v: (* v 2) by: k}`). +;; The sub-table expression (* v 2) returns a vector of the group's size, so +;; each group's result is scattered elementwise back to its rows through +;; idxs[r] — symmetric with the atom broadcast. Regression: this path used to +;; write nothing, leaving the memset'd column all-zero (silent data loss). ;; ──────────────────────────────────────────────────────────────────── (set Tupd_by_vec (table [k v] (list (list "a" "b" "a" "b") [10 20 30 40]))) -;; update by: k where expression (* v 2) returns a vector per group -;; line 8376 fires: ray_is_vec(agg_result) = true for first group -;; (vector result has no broadcast → new_v column filled with 0) +;; NEW column: group "a"=rows 0,2 -> [20 60]; group "b"=rows 1,3 -> [40 80] (count (update {new_v: (* v 2) by: k from: Tupd_by_vec})) -- 4 -;; Vector group result is NOT broadcast → new_v column stays all-zero. -;; Assert the actual modified cells, not just the row count. -(at (update {new_v: (* v 2) by: k from: Tupd_by_vec}) 'new_v) -- [40 80 0 0] +(at (update {new_v: (* v 2) by: k from: Tupd_by_vec}) 'new_v) -- [20 40 60 80] +;; REPLACING an existing column in place must scatter too, not zero it out +(at (update {v: (* v 2) by: k from: Tupd_by_vec}) 'v) -- [20 40 60 80] +;; the reviewer's exact repro: integer keys, an untouched trailing column +(set Tupd_by_vec3 (table [k v w] (list [1 2 1 2] [10 20 30 40] [7 7 7 7]))) +(at (update {v: (* v 2) from: Tupd_by_vec3 by: k}) 'v) -- [20 40 60 80] +(at (update {v: (* v 2) from: Tupd_by_vec3 by: k}) 'w) -- [7 7 7 7] +;; a vector result whose length != the group size has no row mapping -> length error +(update {new_v: [1 2 3] by: k from: Tupd_by_vec}) !- length ;; ──────────────────────────────────────────────────────────────────── ;; update WHERE with LIST-type expression → type error (lines 8682-8684): @@ -463,25 +464,31 @@ ;; ──────────────────────────────────────────────────────────────────── ;; I32 key: case RAY_I32 at lines 131-133 (set Tupd_i32by (table [k v] (list (as 'I32 [1 2 1 2 3]) [10 20 30 40 50]))) -;; update by: scatters aggregate back to original 5 rows (count unchanged) -;; Groups: k=1→sum=40, k=2→sum=60, k=3→sum=50. -;; Scatter fills only first occurrence per group; others remain 0. -;; Row values: [40, 60, 0, 0, 50] → sum = 150 +;; update by: broadcasts the aggregate back to every row of its group +;; (grouped-update semantics — the group sum lands on ALL member rows, not just +;; the first). Groups: k=1→sum=40, k=2→sum=60, k=3→sum=50. +;; Row values: [40, 60, 40, 60, 50] → sum = 250 +;; +;; USER-FACING FIX (was bug): pre-fix, updating an EXISTING column `v` via +;; `by:` silently did nothing — the aggregate was appended as a duplicate +;; column (schema `[k v v]`), so `(at U 'v)` kept reading the stale +;; original [10 20 30 40 50] → sum 150. From a user's perspective the +;; `by:`-update "succeeded" (no error) but the column never changed, and +;; `(key U)` unexpectedly listed the column name twice. (count (update {v: (sum v) by: k from: Tupd_i32by})) -- 5 -(sum (at (update {v: (sum v) by: k from: Tupd_i32by}) 'v)) -- 150 +(sum (at (update {v: (sum v) by: k from: Tupd_i32by}) 'v)) -- 250 ;; BOOL key: case RAY_BOOL (RAY_U8) at lines 135-136 (set Tupd_boolby (table [k v] (list [true false true false] [10 20 30 40]))) -;; Groups: k=true→sum=40, k=false→sum=60. -;; Scatter fills first occurrence; others remain 0: [40, 60, 0, 0] → sum = 100 +;; Groups: k=true→sum=40, k=false→sum=60 → broadcast [40, 60, 40, 60] → sum = 200 (count (update {v: (sum v) by: k from: Tupd_boolby})) -- 4 -(sum (at (update {v: (sum v) by: k from: Tupd_boolby}) 'v)) -- 100 +(sum (at (update {v: (sum v) by: k from: Tupd_boolby}) 'v)) -- 200 ;; F64 key: case RAY_F64 at line 137 (set Tupd_f64by (table [k v] (list [1.0 2.0 1.0 2.0] [10 20 30 40]))) -;; Groups: k=1.0→sum=40, k=2.0→sum=60. [40, 60, 0, 0] → sum = 100 +;; Groups: k=1.0→sum=40, k=2.0→sum=60 → broadcast [40, 60, 40, 60] → sum = 200 (count (update {v: (sum v) by: k from: Tupd_f64by})) -- 4 -(sum (at (update {v: (sum v) by: k from: Tupd_f64by}) 'v)) -- 100 +(sum (at (update {v: (sum v) by: k from: Tupd_f64by}) 'v)) -- 200 ;; ──────────────────────────────────────────────────────────────────── ;; WHERE-update SYM column with null in expr_vec (line 8707) diff --git a/test/rfl/query/take_str_pool.rfl b/test/rfl/query/take_str_pool.rfl new file mode 100644 index 00000000..278e5a3c --- /dev/null +++ b/test/rfl/query/take_str_pool.rfl @@ -0,0 +1,35 @@ +;; Issue #404: `select` with `take:` returned EMPTY strings for pool-backed +;; STR values (longer than the 12-byte inline limit). The DAG OP_HEAD / +;; OP_TAIL executors' flat-column copy adopted the SYM domain but never +;; propagated the source's str_pool, so the sliced descriptors pointed into +;; a pool the result didn't carry. Inline (≤12 byte) strings were unaffected, +;; which is why this could hide in tables of short strings. + +(set Tsp (table [s v] (list ["long-string-value-zero" "long-string-value-one" "long-string-value-two"] [0 1 2]))) + +;; baseline: unlimited select keeps pooled strings (always worked) +(at (at (select {from: Tsp s: s where: (> v 0)}) 's) 0) -- "long-string-value-one" + +;; identity select + take (OP_HEAD, no filter) +(at (at (select {from: Tsp take: 1}) 's) 0) -- "long-string-value-zero" + +;; projection + take +(at (at (select {from: Tsp s: s take: 1}) 's) 0) -- "long-string-value-zero" + +;; where + take (lazy selection compacted, then OP_HEAD) +(at (at (select {from: Tsp s: s where: (> v 0) take: 1}) 's) 0) -- "long-string-value-one" + +;; negative take → OP_TAIL +(at (at (select {from: Tsp take: -1}) 's) 0) -- "long-string-value-two" +(at (at (select {from: Tsp s: s where: (< v 2) take: -1}) 's) 0) -- "long-string-value-one" + +;; take larger than nrows → clamp, strings intact +(at (at (select {from: Tsp take: 100}) 's) 2) -- "long-string-value-two" + +;; sort + take still correct (was already working — pin it) +(at (at (select {from: Tsp s: s where: (> v 0) asc: s take: 1}) 's) 0) -- "long-string-value-one" + +;; inline strings (≤12 bytes) through the same paths stay correct +(set Tsi (table [s v] (list ["aa" "bb" "cc"] [0 1 2]))) +(at (at (select {from: Tsi take: 1}) 's) 0) -- "aa" +(at (at (select {from: Tsi take: -1}) 's) 0) -- "cc" diff --git a/test/rfl/query/ungrouped_count_distinct.rfl b/test/rfl/query/ungrouped_count_distinct.rfl new file mode 100644 index 00000000..4e496756 --- /dev/null +++ b/test/rfl/query/ungrouped_count_distinct.rfl @@ -0,0 +1,41 @@ +;; Issue #405: ungrouped `(count (distinct col))` as a select projection did +;; not compute a distinct count. The scalar-aggregate eval fallback +;; (eval_scalar_agg_outputs) evaluated the aggregate's argument per row, so +;; `distinct` saw one scalar cell at a time: STR/SYM rows collapsed into a +;; row-count answer (silently wrong), I64 rows errored ("argument must be a +;; list"). A whole-column argument (distinct/asc/desc/reverse at its head) +;; must be evaluated once against the full column, like the projection +;; fallback already does. + +(set Tcd (table [i s g] (list [1 1 2 2 3] ["aa" "aa" "bb" "bb" "cc"] [9 9 9 9 9]))) + +;; whole-vector reducer — always worked; pin as the reference +(count (distinct (at Tcd 'i))) -- 3 +(count (distinct (at Tcd 's))) -- 3 + +;; grouped form — always worked; pin it +(at (at (select {from: Tcd by: g u: (count (distinct i))}) 'u) 0) -- 3 +(at (at (select {from: Tcd by: g u: (count (distinct s))}) 'u) 0) -- 3 + +;; ungrouped select projection: was 5 (row count) for STR, error for I64 +(at (at (select {from: Tcd u: (count (distinct s))}) 'u) 0) -- 3 +(at (at (select {from: Tcd u: (count (distinct i))}) 'u) 0) -- 3 + +;; single-group result stays one row +(count (select {from: Tcd u: (count (distinct s))})) -- 1 + +;; with a where: filter — distinct over the filtered rows only +(at (at (select {from: Tcd u: (count (distinct i)) where: (< i 3)}) 'u) 0) -- 2 + +;; multiple aggregate outputs side by side +(set Rcd (select {from: Tcd u: (count (distinct s)) n: (count s)})) +(at (at Rcd 'u) 0) -- 3 +(at (at Rcd 'n) 0) -- 5 + +;; pool-backed (>12 byte) strings through the same path +(set Tcw (table [s] (list ["long-string-value-zero" "long-string-value-zero" "long-string-value-one"]))) +(at (at (select {from: Tcw u: (count (distinct s))}) 'u) 0) -- 2 + +;; other whole-column verbs as aggregate arguments +(at (at (select {from: Tcd u: (count (reverse i))}) 'u) 0) -- 5 +(at (at (select {from: Tcd u: (sum (distinct i))}) 'u) 0) -- 6 diff --git a/test/rfl/query/update_parted.rfl b/test/rfl/query/update_parted.rfl new file mode 100644 index 00000000..3510cc22 --- /dev/null +++ b/test/rfl/query/update_parted.rfl @@ -0,0 +1,99 @@ +;; Regression for `update` over PARTED tables (src/ops/query.c). +;; +;; Two distinct bugs, both hitting a `.db.parted.get` table as the `from:` +;; source: +;; +;; 1. PARTED columns carry the RAY_PARTED_BASE wrapper type (printed as "?") +;; and a RAY_MAPCOMMON partition key. ray_update read the original +;; column through `ray_vec_new(ct, ...)` / `ray_data(col)` / the grouped +;; gather, none of which understood the segmented shape — so MODIFYING an +;; existing column of a parted table failed with +;; `expression type I64 does not match ? column` (and `by:` with +;; `group: argument must be a vector`). Fix: flatten a parted input +;; table once, the way `select` does. +;; +;; 2. The `by:`-UPDATE branch duplicated an existing target column instead +;; of replacing it — `(update {w: (sum v) from: T by: k})` on a table that +;; already has `w` produced schema `[k v w w]` and `at` read the stale +;; original. Affected flat tables too. Fix: substitute existing target +;; columns in their original slots and append only new columns. +;; +;; Both are checked against flat-table oracles with identical data. + +;; ────────────── build a 2-partition parted table ────────────── +(.sys.exec "rm -rf /tmp/rfl_update_parted") +(set D1 (table [k v w] (list [1 2 3] [10 20 30] [100 200 300]))) +(set D2 (table [k v w] (list [1 1 2] [40 50 60] [400 500 600]))) +(.db.splayed.set "/tmp/rfl_update_parted/2024.01.01/t/" D1) +(.db.splayed.set "/tmp/rfl_update_parted/2024.01.02/t/" D2) +(set Pt (.db.parted.get "/tmp/rfl_update_parted/" 't)) +(set flat (table [k v w] (list [1 2 3 1 1 2] [10 20 30 40 50 60] [100 200 300 400 500 600]))) + +;; ────────────── fix 1a: modify existing column, no where ────────────── +;; USER-FACING: pre-fix, `(update {v: (+ v 100) from: Pt})` on a parted table +;; aborted with an immediate type error — +;; `error: type: update: expression type I64 does not match ? column` +;; so the user could not modify ANY existing column of a parted table at all. +(at (update {v: (+ v 100) from: Pt}) 'v) -- [110 120 130 140 150 160] +(count (update {v: (+ v 100) from: Pt})) -- 6 + +;; ────────────── fix 1b: scalar broadcast into existing column ────────────── +;; USER-FACING: pre-fix, `(update {v: 5 from: Pt})` failed with the same +;; `expression type I64 does not match ? column` — even a plain constant +;; cannot be written over an existing parted column. +(at (update {v: 5 from: Pt}) 'v) -- [5 5 5 5 5 5] + +;; ────────────── fix 1c: where-masked update of existing column ────────────── +;; USER-FACING: pre-fix, `where:`-masked writes failed with +;; `error: type: vec_new: type must be a positive concrete vector type, got ?` +;; so conditional in-place updates over parted data were impossible. +(at (update {v: 99 from: Pt where: (> k 1)}) 'v) -- [10 99 99 40 50 99] + +;; ────────────── fix 1d: update by: aggregate broadcast over parted ────────────── +;; USER-FACING: pre-fix, `by:`-grouped updates on parted tables errored with +;; `error: type: group: argument must be a vector or list, got ?` +;; while on FLAT tables (bug 2) the write silently appeared to do nothing: +;; the aggregate was appended as a duplicate column, the query "succeeded", +;; `(key U)` suddenly listed the column twice, and `(at U 'w)` kept returning +;; the stale original values. +;; k=1 row v={10,40,50} sum=100; k=2 {20,60} sum=80; k=3 {30} sum=30 +(at (update {w: (sum v) from: Pt by: k}) 'w) -- [100 80 30 100 100 80] +;; fix 2: schema must NOT duplicate the replaced target column +(key (update {w: (sum v) from: Pt by: k})) -- [date k v w] +;; flat oracle — same rows, same answers +(at (update {w: (sum v) from: flat by: k}) 'w) -- [100 80 30 100 100 80] +(key (update {w: (sum v) from: flat by: k})) -- [k v w] +;; replaced columns keep their original schema position, even when not last +(key (update {v: (sum v) from: flat by: k})) -- [k v w] +(at (update {v: (sum v) from: flat by: k}) 'v) -- [100 80 30 100 100 80] +;; empty grouped updates preserve an existing target column's type +(set E (table [k v w] (list (as 'I64 []) (as 'F64 []) (as 'F64 [])))) +(key (update {w: (sum v) from: E by: k})) -- [k v w] +(type (at (update {w: (sum v) from: E by: k}) 'w)) -- 'F64 + +;; ────────────── fix 1e: mixed update — modify existing + add new over parted ────────────── +;; USER-FACING: any dict naming an EXISTING column tripped the fix-1a error +;; even when it also added new columns (`z`), so mixed single-pass updates +;; over parted tables were not possible. +(set U5 (update {v: (+ v 1) z: (* k 10) from: Pt})) +(at U5 'v) -- [11 21 31 41 51 61] +(at U5 'z) -- [10 20 30 10 10 20] +(key U5) -- [date k v w z] + +;; ────────────── in-place amend of a parted global (from: 'name) ────────────── +;; NOTE: `(update {from: 'G …})` amends the env global G in place and returns a +;; SYM (it does not return a new table) — for both flat and parted, so the +;; caller ignores the return and inspects G afterwards. +;; USER-FACING: pre-fix, even this over a parted global hit the same +;; `does not match ? column` error. +(set G (.db.parted.get "/tmp/rfl_update_parted/" 't)) +(update {from: 'G v: (* v 100)}) ;; amend G in place, ignore the sym return +(at G 'v) -- [1000 2000 3000 4000 5000 6000] +(key G) -- [date k v w] +;; flat oracle — same rows, same answer (in-place amend via symbol) +(set Gflat (table [k v w] (list [1 2 3 1 1 2] [10 20 30 40 50 60] [100 200 300 400 500 600]))) +(update {from: 'Gflat v: (* v 100)}) +(at Gflat 'v) -- [1000 2000 3000 4000 5000 6000] +(key Gflat) -- [k v w] + +(.sys.exec "rm -rf /tmp/rfl_update_parted") diff --git a/test/rfl/system/db_parted_fill.rfl b/test/rfl/system/db_parted_fill.rfl index a1466b05..2574b271 100644 --- a/test/rfl/system/db_parted_fill.rfl +++ b/test/rfl/system/db_parted_fill.rfl @@ -40,7 +40,27 @@ ;; Idempotent: a second fill finds nothing missing → empty result. (.db.parted.fill "/tmp/rfl_fill/") -- [] +;; Regression #401: an empty copy of a LIST column must use the LIST +;; constructor, not the concrete-vector constructor (RAY_LIST == 0). +(set SCHED (table [acct sched] (list ['a 'b] (list (dict [0] [5.0]) (dict [0 250000] [15.0 10.0]))))) +(.db.splayed.set "/tmp/rfl_fill/2024.01.03/SCHED/" SCHED) +(.db.parted.fill "/tmp/rfl_fill/") -- ['2024.01.01 '2024.01.02] +(set RS (.db.parted.get "/tmp/rfl_fill/" 'SCHED)) +(count RS) -- 2 +(type (at RS 'sched)) -- 'LIST +(type (at (at RS 'sched) 0)) -- 'DICT +(at (at (at RS 'sched) 1) 250000) -- 10.0 +(.db.parted.fill "/tmp/rfl_fill/") -- [] + +;; Template errors retain their real code instead of being flattened to io. +(.sys.exec "rm -rf /tmp/rfl_fill_bad") -- 0 +(set TN (table [v] (list [1]))) +(.db.splayed.set "/tmp/rfl_fill_bad/2024.01.01/A/" TN) +(.db.splayed.set "/tmp/rfl_fill_bad/2024.01.02/BAD/" TN) +(.sys.exec "printf 'bad' > /tmp/rfl_fill_bad/2024.01.02/BAD/.d") -- 0 +(.db.parted.fill "/tmp/rfl_fill_bad/") !- corrupt + ;; Error paths: missing root and a non-parted (splayed) root. (.db.parted.fill "/tmp/rfl_fill_nope/") !- io -(.sys.exec "rm -rf /tmp/rfl_fill") +(.sys.exec "rm -rf /tmp/rfl_fill /tmp/rfl_fill_bad") diff --git a/test/rfl/temporal/date.rfl b/test/rfl/temporal/date.rfl index 06aaf382..b4ae2a5a 100644 --- a/test/rfl/temporal/date.rfl +++ b/test/rfl/temporal/date.rfl @@ -43,3 +43,25 @@ (== 2024.06.15 2024.06.15) -- true (<= 2024.01.01 2024.01.01) -- true (>= 2024.12.31 2024.06.15) -- true + +;; ────────────── int32-range decode (regression) ────────────── +;; days near INT32_MAX used to overflow int32 in date_to_ymd (cal.h:50) +;; and print garbage / trip UBSan. Years above 9999 can't be written as +;; literals, so pin decode+re-encode identity instead. +(as 'DATE 0) -- 2000.01.01 +(as 'DATE -10957) -- 1970.01.01 +(as 'DATE 2921939) -- 9999.12.31 +(as 'i64 (as 'DATE 2146753528)) -- 2146753528 +(as 'i64 (as 'DATE 2146753529)) -- 2146753529 +(as 'i64 (as 'DATE 2147483646)) -- 2147483646 +(as 'i64 (as 'DATE 2147483647)) -- 2147483647 +(as 'i64 (+ (as 'DATE 2146753528) 1)) -- 2146753529 +(as 'i64 (as 'DATE -2147483647)) -- -2147483647 +(as 'DATE -2147483648) -- 0Nd +;; round-trip through the int64 store stays exact +(as 'i64 2000.01.01) -- 0 +(as 'i64 1970.01.01) -- -10957 +(as 'i64 2024.02.29) -- 8825 +(as 'i64 1900.02.28) -- -36466 +(as 'i64 9999.12.31) -- 2921939 +(as 'i64 2000.02.29) -- 59 diff --git a/test/rfl/temporal/parallel_extract.rfl b/test/rfl/temporal/parallel_extract.rfl new file mode 100644 index 00000000..1984e6cc --- /dev/null +++ b/test/rfl/temporal/parallel_extract.rfl @@ -0,0 +1,123 @@ +;; Parallel temporal extract / truncate — src/ops/temporal.c +;; +;; Both kernel families chunk their row range over the thread pool once the +;; column reaches RAY_PARALLEL_THRESHOLD (65 536 rows): +;; * ray_temporal_extract / ray_temporal_truncate — the eval builtins +;; (minute, ss, hh, yyyy, mm, date, time), +;; * exec_extract / exec_date_trunc — the DAG ops, reached by writing +;; (minute ts) / ts.date inside a `select`. +;; +;; Nulls here are payload sentinels, not a bitmap, so the per-row write is +;; already disjoint across workers. The one piece of shared state is the +;; HAS_NULLS attr byte: workers never touch it, they raise one atomic flag +;; and the caller folds it into the result after the dispatch joins. Note +;; that per-element `nil?` tests the sentinel bit pattern and NEVER reaches +;; the vec-level HAS_NULLS gate — only whole-vector consumers (min/max/avg, +;; group-by) do, which is why the min/max assertions below exist: they are +;; the ones that fail if the fold is dropped (min then reads the sentinel +;; as a value and returns 0Nl). +;; +;; Null placement is deliberate: index 163840 is both 64-element aligned and +;; exactly 20 x the 8192-row dispatch grain, i.e. an interior task seam, and +;; 163843 is mid-word inside the following task. A chunking bug lands on +;; one of the two. +;; +;; Row i of every column below is i seconds after 2000.01.01, so every +;; expected value is closed-form over (til N) and nothing is hand-computed. + +;; ════════════════════════════════════════════════════════════════════ +;; Section 1: >150K rows (parallel), no nulls +;; ════════════════════════════════════════════════════════════════════ + +(set N 200005) +(set NF (as 'TIMESTAMP (* (til N) 1000000000))) +(count NF) -- 200005 + +(all (== (minute NF) (% (div (til N) 60) 60))) -- true +(all (== (ss NF) (% (til N) 60))) -- true +(all (== (hh NF) (% (div (til N) 3600) 24))) -- true + +;; 200 005 s is 2.3 days, so the whole column is January 2000. +(all (== (yyyy NF) 2000)) -- true +(all (== (mm NF) 1)) -- true + +;; ray_temporal_truncate, DAY bucket, over the same 200K rows. +(all (== (date NF) (as 'TIMESTAMP (* (div (til N) 86400) 86400000000000)))) -- true + +;; ════════════════════════════════════════════════════════════════════ +;; Section 2: parallel rows with nulls on a task seam and mid-word +;; ════════════════════════════════════════════════════════════════════ +;; Indices 163840 and 163843 are null; 163841/163842/163844 are not. + +(set NB_HEAD (as 'TIMESTAMP (* (til 163840) 1000000000))) +(set NB_SEAM (as 'TIMESTAMP [0N 163841000000000 163842000000000 0N 163844000000000])) +(set NB_TAIL (as 'TIMESTAMP (* (+ (til 36160) 163845) 1000000000))) +(set NB (concat (concat NB_HEAD NB_SEAM) NB_TAIL)) +(count NB) -- 200005 + +;; The two nulls survive as nulls. +(nil? (at (minute NB) 163840)) -- true +(nil? (at (minute NB) 163843)) -- true +(nil? (at (ss NB) 163840)) -- true +(nil? (at (ss NB) 163843)) -- true + +;; The HAS_NULLS fold itself: these consumers hit the vec-level gate. +;; With the fold dropped, min reads the sentinel as a value (0Nl). +(min (minute NB)) -- 0 +(max (minute NB)) -- 59 +(nil? (min (ss NB))) -- false +(min (ss NB)) -- 0 + +;; Their non-null neighbours, on both sides of the seam, are untouched. +(at (ss NB) 163839) -- 39 +(at (ss NB) 163841) -- 41 +(at (ss NB) 163842) -- 42 +(at (ss NB) 163844) -- 44 +(nil? (at (ss NB) 163839)) -- false +(nil? (at (ss NB) 163844)) -- false + +;; Everything before and after the null run matches the closed form. +(all (== (take (minute NB) 163840) (% (div (til 163840) 60) 60))) -- true +(all (== (drop (minute NB) 163845) (% (div (+ (til 36160) 163845) 60) 60))) -- true + +;; Truncate over the same nullable column. +(nil? (at (date NB) 163843)) -- true +(at (date NB) 163841) -- 2000.01.02D00:00:00.000000000 + +;; ════════════════════════════════════════════════════════════════════ +;; Section 3: the DAG ops (exec_extract / exec_date_trunc) +;; ════════════════════════════════════════════════════════════════════ + +(set TF (table [ts] (list NF))) +(set TN (table [ts] (list NB))) + +(all (== (at (select {m: (minute ts) from: TF}) 'm) (% (div (til N) 60) 60))) -- true +(all (== (at (select {s: (second ts) from: TF}) 's) (% (til N) 60))) -- true +(all (== (at (select {h: (hour ts) from: TF}) 'h) (% (div (til N) 3600) 24))) -- true + +(nil? (at (at (select {s: (second ts) from: TN}) 's) 163840)) -- true +(nil? (at (at (select {s: (second ts) from: TN}) 's) 163843)) -- true +(at (at (select {s: (second ts) from: TN}) 's) 163841) -- 41 +(at (at (select {s: (second ts) from: TN}) 's) 163844) -- 44 + +;; DAG-path HAS_NULLS fold, same argument as Section 2. +(min (at (select {s: (second ts) from: TN}) 's)) -- 0 + +(nil? (at (at (select {s: ts.date from: TN}) 's) 163840)) -- true +(at (at (select {s: ts.time from: TN}) 's) 163841) -- 2000.01.02D21:30:41.000000000 + +;; ════════════════════════════════════════════════════════════════════ +;; Section 4: sub-threshold columns stay serial and must agree +;; ════════════════════════════════════════════════════════════════════ + +(set SM (as 'TIMESTAMP (* (til 1000) 1000000000))) +(all (== (minute SM) (% (div (til 1000) 60) 60))) -- true +(all (== (ss SM) (% (til 1000) 60))) -- true + +;; Null exactly on a 64-element boundary in the serial path. +(set SN (concat (as 'TIMESTAMP (* (til 64) 1000000000)) (as 'TIMESTAMP [0N 65000000000]))) +(count SN) -- 66 +(nil? (at (minute SN) 64)) -- true +(at (ss SN) 63) -- 3 +(at (minute SN) 65) -- 1 +(nil? (at (minute SN) 63)) -- false diff --git a/test/test_group_extra.c b/test/test_group_extra.c index 3b3154a2..bfccd310 100644 --- a/test/test_group_extra.c +++ b/test/test_group_extra.c @@ -47,8 +47,10 @@ #include "ops/ops.h" #include "ops/internal.h" #include "ops/hll.h" +#include "ops/cdfuse.h" #include "table/sym.h" #include +#include #include #define N 70000 /* > RAY_PARALLEL_THRESHOLD (65536) */ @@ -1078,6 +1080,11 @@ static test_result_t test_i16_group_top_count_emit_filter(void) { filter.enabled = 1; filter.agg_index = 0; filter.top_count_take = 2; + /* Direction is explicit since issue #408: the keep-min count trims are + * largest-first machinery, so they run for desc = 1 only (an asc take + * keeps the SMALLEST N and is served by the bounded heap / the + * downstream sort+take instead). */ + filter.desc = 1; ray_group_emit_filter_set(filter); ray_t* res = ray_execute(g, grp); ray_group_emit_filter_set(prev); @@ -1098,6 +1105,33 @@ static test_result_t test_i16_group_top_count_emit_filter(void) { if (k == 2 && c == 4) got_2 = 1; } TEST_ASSERT_TRUE(got_1 && got_2); + ray_release(res); + + /* Same filter with desc = 0 (`asc: c take: 2`): the emit filter must NOT + * keep the two LARGEST groups. Trimming here is desc-only machinery, so + * the asc request falls through to the full group set and the caller's + * sort+take picks the smallest — what must never happen is the result + * losing the small groups (issue #408: asc returned the desc answer). */ + filter.desc = 0; + ray_group_emit_filter_set(filter); + res = ray_execute(g, grp); + ray_group_emit_filter_set(prev); + TEST_ASSERT_FALSE(RAY_IS_ERR(res)); + out_key = ray_table_get_col(res, key_sym); + out_cnt = ray_table_get_col_idx(res, 1); + TEST_ASSERT_NOT_NULL(out_key); + TEST_ASSERT_NOT_NULL(out_cnt); + /* The asc request falls through to the FULL group set (all 5 groups) — + * pin the row count too, so this can tell "full fall-through" apart + * from a hypothetical asc top-2, and cannot pass by accident. */ + TEST_ASSERT_EQ_I(ray_table_nrows(res), 5); + int got_smallest = 0; + for (int64_t i = 0; i < ray_table_nrows(res); i++) { + int16_t k = ((int16_t*)ray_data(out_key))[i]; + int64_t c = ((int64_t*)ray_data(out_cnt))[i]; + if (k == 5 && c == 1) got_smallest = 1; + } + TEST_ASSERT_TRUE(got_smallest); ray_release(res); ray_graph_free(g); @@ -1147,6 +1181,11 @@ static test_result_t test_sym_group_top_count_emit_filter(void) { filter.enabled = 1; filter.agg_index = 0; filter.top_count_take = 2; + /* Direction is explicit since issue #408: the keep-min count trims are + * largest-first machinery, so they run for desc = 1 only (an asc take + * keeps the SMALLEST N and is served by the bounded heap / the + * downstream sort+take instead). */ + filter.desc = 1; ray_group_emit_filter_set(filter); ray_t* res = ray_execute(g, grp); ray_group_emit_filter_set(prev); @@ -1223,6 +1262,11 @@ static test_result_t test_five_key_group_top_count_emit_filter(void) { filter.enabled = 1; filter.agg_index = 0; filter.top_count_take = 2; + /* Direction is explicit since issue #408: the keep-min count trims are + * largest-first machinery, so they run for desc = 1 only (an asc take + * keeps the SMALLEST N and is served by the bounded heap / the + * downstream sort+take instead). */ + filter.desc = 1; ray_group_emit_filter_set(filter); ray_t* res = ray_execute(g, grp); ray_group_emit_filter_set(prev); @@ -2320,7 +2364,8 @@ static test_result_t test_ght_layout_copy_depth_invariance(void) { ght_layout_t master; TEST_ASSERT_TRUE(ght_compute_layout(&master, NK, NA, agg_vecs, NULL, - GHT_NEED_SUM, agg_ops, key_types)); + GHT_NEED_SUM, agg_ops, key_types, + NULL)); /* > GHT_INLINE on both axes: this must be a real, owned spill block. */ TEST_ASSERT_NOT_NULL(master.spill_hdr); TEST_ASSERT_FALSE(master.agg_val_slot == master.agg_val_slot_in); @@ -2376,7 +2421,8 @@ static test_result_t test_ght_layout_copy_depth_invariance(void) { * never left aliasing the source's — the mirror of the spill leg above. */ ght_layout_t inl; TEST_ASSERT_TRUE(ght_compute_layout(&inl, 2, 2, agg_vecs, NULL, - GHT_NEED_SUM, agg_ops, key_types)); + GHT_NEED_SUM, agg_ops, key_types, + NULL)); TEST_ASSERT_NULL(inl.spill_hdr); TEST_ASSERT_TRUE(inl.agg_val_slot == inl.agg_val_slot_in); ght_layout_t ic; @@ -2404,6 +2450,285 @@ static test_result_t test_ght_layout_copy_depth_invariance(void) { PASS(); } +/* -------------------------------------------------------------------------- + * Packed-key eligibility: F64 keys must never join the packed lane path. + * read_col_i64 — which the packed stagers use to load every lane — does not + * decode F64, so a packed F64 key would hash and compare the wrong bits; the + * exclusion also keeps the packed lanes clear of the -0.0 canonicalisation + * the F64 key builders apply (group_key_f64_bits, #407). No rfl-level + * assertion can pin the predicate itself (every layout now agrees on the + * ±0.0 answer), so it is asserted directly here. + * -------------------------------------------------------------------------- */ +static test_result_t test_ght_packed_key_excludes_f64(void) { + ray_heap_init(); + (void)ray_sym_init(); + + uint16_t agg_ops[1] = { OP_SUM }; + ray_t* agg_vecs[1] = { ray_vec_new(RAY_F64, 1) }; + TEST_ASSERT_NOT_NULL(agg_vecs[0]); + agg_vecs[0]->len = 1; + ((double*)ray_data(agg_vecs[0]))[0] = 0.0; + + int8_t key_types[2] = { RAY_I64, RAY_I64 }; + ght_layout_t ly; + TEST_ASSERT_TRUE(ght_compute_layout(&ly, 2, 1, agg_vecs, NULL, + GHT_NEED_SUM, agg_ops, key_types, + NULL)); + TEST_ASSERT_TRUE(ly.packed_key); /* integer keys: packed */ + ght_layout_free(&ly); + + key_types[1] = RAY_F64; + TEST_ASSERT_TRUE(ght_compute_layout(&ly, 2, 1, agg_vecs, NULL, + GHT_NEED_SUM, agg_ops, key_types, + NULL)); + TEST_ASSERT_FALSE(ly.packed_key); /* any F64 key: excluded */ + ght_layout_free(&ly); + + ray_release(agg_vecs[0]); + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + +/* -------------------------------------------------------------------------- + * Fused grouped count-distinct kernel (src/ops/cdfuse.c) + * -------------------------------------------------------------------------- */ + +/* Full O(n) reference check of a ray_cd_fused result against the row-major + * key/value arrays the caller generated. Verifies, for EVERY group: the key + * (in first-seen order), the distinct count, and the _first row — plus that + * the emitted _first column is strictly increasing. Key values must lie in + * [0,nk), value values in [0,nv). */ +static test_result_t cdf_verify(ray_t* r, const int64_t* kref, const int64_t* vref, + int64_t n, int64_t nk, int64_t nv) { + TEST_ASSERT_NOT_NULL(r); + TEST_ASSERT(r->type == RAY_TABLE, "fused cd returns a table"); + + int64_t* exp_cnt = (int64_t*)calloc((size_t)nk, sizeof(int64_t)); + int64_t* exp_first = (int64_t*)malloc((size_t)nk * sizeof(int64_t)); + int64_t* exp_key = (int64_t*)malloc((size_t)nk * sizeof(int64_t)); + char* seen = (char*)calloc((size_t)nk * (size_t)nv, 1); + TEST_ASSERT_NOT_NULL(exp_cnt); + TEST_ASSERT_NOT_NULL(exp_first); + TEST_ASSERT_NOT_NULL(exp_key); + TEST_ASSERT_NOT_NULL(seen); + for (int64_t i = 0; i < nk; i++) exp_first[i] = -1; + + int64_t exp_ng = 0; + for (int64_t i = 0; i < n; i++) { + int64_t kk = kref[i], vv = vref[i]; + if (exp_first[kk] < 0) { exp_first[kk] = i; exp_key[exp_ng++] = kk; } + char* slot = &seen[kk * nv + vv]; + if (!*slot) { *slot = 1; exp_cnt[kk]++; } + } + + ray_t* keys = ray_table_get_col_idx(r, 0); + ray_t* cnts = ray_table_get_col_idx(r, 1); + ray_t* firsts = ray_table_get_col_idx(r, 2); + TEST_ASSERT_NOT_NULL(keys); + TEST_ASSERT_NOT_NULL(cnts); + TEST_ASSERT_NOT_NULL(firsts); + TEST_ASSERT_EQ_I(ray_table_nrows(r), exp_ng); + const int64_t* gk = (const int64_t*)ray_data(keys); + const int64_t* gc = (const int64_t*)ray_data(cnts); + const int64_t* gf = (const int64_t*)ray_data(firsts); + for (int64_t g = 0; g < exp_ng; g++) { + TEST_ASSERT_FMT(gk[g] == exp_key[g], + "group %lld key: got %lld, expected %lld", + (long long)g, (long long)gk[g], (long long)exp_key[g]); + TEST_ASSERT_FMT(gc[g] == exp_cnt[exp_key[g]], + "group %lld (key %lld) count: got %lld, expected %lld", + (long long)g, (long long)exp_key[g], (long long)gc[g], + (long long)exp_cnt[exp_key[g]]); + TEST_ASSERT_FMT(gf[g] == exp_first[exp_key[g]], + "group %lld (key %lld) _first: got %lld, expected %lld", + (long long)g, (long long)exp_key[g], (long long)gf[g], + (long long)exp_first[exp_key[g]]); + TEST_ASSERT_FMT(g == 0 || gf[g] > gf[g - 1], + "_first not strictly increasing at group %lld", (long long)g); + } + free(exp_cnt); free(exp_first); free(exp_key); free(seen); + PASS(); +} + +static test_result_t test_cd_fused_basic(void) { + ray_heap_init(); + (void)ray_sym_init(); + + /* 300000 rows, 1000 keys, 3 distinct values per key (see the rfl pin). + * Row count is >= CDF_MIN_ROWS so the kernel's admission gate passes; + * the key/value shape mirrors the brief's 200-key sketch. */ + int64_t n = 300000, nk = 1000, nv = 3000; + ray_t* k = ray_vec_new(RAY_I64, n); + ray_t* v = ray_vec_new(RAY_I64, n); + TEST_ASSERT_NOT_NULL(k); + TEST_ASSERT_NOT_NULL(v); + int64_t* kd = (int64_t*)ray_data(k); + int64_t* vd = (int64_t*)ray_data(v); + for (int64_t i = 0; i < n; i++) { kd[i] = i % nk; vd[i] = i % nv; } + k->len = n; v->len = n; + + ray_t* r = ray_cd_fused(k, v, n); + TEST_ASSERT_NOT_NULL(r); + TEST_ASSERT_EQ_I(ray_table_nrows(r), nk); + /* stable first-seen order: key i at row i for this data */ + TEST_ASSERT_EQ_I(((int64_t*)ray_data(ray_table_get_col_idx(r, 0)))[0], 0); + TEST_ASSERT_EQ_I(((int64_t*)ray_data(ray_table_get_col_idx(r, 0)))[nk - 1], nk - 1); + test_result_t chk = cdf_verify(r, kd, vd, n, nk, nv); + if (chk.status != TEST_PASS) return chk; + int64_t total = 0; + for (int64_t g = 0; g < nk; g++) + total += ((int64_t*)ray_data(ray_table_get_col_idx(r, 1)))[g]; + TEST_ASSERT_EQ_I(total, nk * 3); /* 3 distinct values per key */ + ray_release(r); ray_release(k); ray_release(v); + + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + +/* Interleaved keys: first occurrences are NOT monotonic in key value, so the + * emitted order genuinely exercises the cross-partition first_row sort and the + * per-record first_row MIN. Distinct counts differ between groups. */ +static test_result_t test_cd_fused_interleaved(void) { + ray_heap_init(); + (void)ray_sym_init(); + + int64_t n = 300000, nk = 997, nv = 1013; + ray_t* k = ray_vec_new(RAY_I64, n); + ray_t* v = ray_vec_new(RAY_I64, n); + TEST_ASSERT_NOT_NULL(k); + TEST_ASSERT_NOT_NULL(v); + int64_t* kd = (int64_t*)ray_data(k); + int64_t* vd = (int64_t*)ray_data(v); + for (int64_t i = 0; i < n; i++) { + kd[i] = (i * 391) % nk; /* 391 coprime with 997 → scrambled order */ + vd[i] = (i * 7) % nv; + } + k->len = n; v->len = n; + + ray_t* r = ray_cd_fused(k, v, n); + TEST_ASSERT_NOT_NULL(r); + test_result_t chk = cdf_verify(r, kd, vd, n, nk, nv); + if (chk.status != TEST_PASS) return chk; + /* keys are emitted in first-seen (not sorted) order for this pattern */ + const int64_t* gk = (const int64_t*)ray_data(ray_table_get_col_idx(r, 0)); + int sorted = 1; + for (int64_t g = 1; g < ray_table_nrows(r); g++) + if (gk[g] < gk[g - 1]) { sorted = 0; break; } + TEST_ASSERT(!sorted, "emitted key order must be first-seen, not sorted"); + ray_release(r); ray_release(k); ray_release(v); + + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + +/* q08 shape: low-cardinality key (200), heavy skew (key 0 owns half the rows), + * high-cardinality values. Key-hash partitioning serialized phase 2 on the + * fat key here; pair-hash partitioning must still produce the exact answer. */ +static test_result_t test_cd_fused_skewed(void) { + ray_heap_init(); + (void)ray_sym_init(); + + int64_t n = 300000, nk = 200, nv = 50021; + ray_t* k = ray_vec_new(RAY_I64, n); + ray_t* v = ray_vec_new(RAY_I64, n); + TEST_ASSERT_NOT_NULL(k); + TEST_ASSERT_NOT_NULL(v); + int64_t* kd = (int64_t*)ray_data(k); + int64_t* vd = (int64_t*)ray_data(v); + for (int64_t i = 0; i < n; i++) { + /* every other row goes to key 0 → key 0 owns 50% of the table */ + kd[i] = (i & 1) ? 1 + ((i / 2) % (nk - 1)) : 0; + vd[i] = (i * 2654435761u) % nv; /* high-cardinality values */ + } + k->len = n; v->len = n; + + ray_t* r = ray_cd_fused(k, v, n); + TEST_ASSERT_NOT_NULL(r); + TEST_ASSERT_EQ_I(ray_table_nrows(r), nk); + test_result_t chk = cdf_verify(r, kd, vd, n, nk, nv); + if (chk.status != TEST_PASS) return chk; + ray_release(r); ray_release(k); ray_release(v); + + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + +/* Self-correlated columns: v == k on every row. A symmetric pair-hash combine + * (hash(k) ^ hash(v)) cancels to zero here, sending every row to partition 0 at + * dedupe slot 0 — a single serial probe cluster that took minutes on 20M rows. + * At 300K rows the query completes either way, so this test guards the SHAPE's + * correctness while the asymmetric combine in cdf_p1_fn guards the cliff. */ +static test_result_t test_cd_fused_self(void) { + ray_heap_init(); + (void)ray_sym_init(); + + int64_t n = 300000, nk = 200; + ray_t* k = ray_vec_new(RAY_I64, n); + ray_t* v = ray_vec_new(RAY_I64, n); + TEST_ASSERT_NOT_NULL(k); + TEST_ASSERT_NOT_NULL(v); + int64_t* kd = (int64_t*)ray_data(k); + int64_t* vd = (int64_t*)ray_data(v); + for (int64_t i = 0; i < n; i++) { kd[i] = i % nk; vd[i] = kd[i]; } + k->len = n; v->len = n; + + ray_t* r = ray_cd_fused(k, v, n); + TEST_ASSERT_NOT_NULL(r); + TEST_ASSERT_EQ_I(ray_table_nrows(r), nk); + test_result_t chk = cdf_verify(r, kd, vd, n, nk, nk); + if (chk.status != TEST_PASS) return chk; + /* v == k → exactly one distinct value per key */ + const int64_t* gc = (const int64_t*)ray_data(ray_table_get_col_idx(r, 1)); + for (int64_t g = 0; g < nk; g++) TEST_ASSERT_EQ_I(gc[g], 1); + ray_release(r); ray_release(k); ray_release(v); + + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + +/* Narrow column types: I32 key, I16 value (read_col_i64 widening path). */ +static test_result_t test_cd_fused_narrow_types(void) { + ray_heap_init(); + (void)ray_sym_init(); + + int64_t n = 300000, nk = 1000, nv = 300; + ray_t* k = ray_vec_new(RAY_I32, n); + ray_t* v = ray_vec_new(RAY_I16, n); + ray_t* kr = ray_vec_new(RAY_I64, n); + ray_t* vr = ray_vec_new(RAY_I64, n); + TEST_ASSERT_NOT_NULL(k); + TEST_ASSERT_NOT_NULL(v); + TEST_ASSERT_NOT_NULL(kr); + TEST_ASSERT_NOT_NULL(vr); + int32_t* kd = (int32_t*)ray_data(k); + int16_t* vd = (int16_t*)ray_data(v); + int64_t* kd64 = (int64_t*)ray_data(kr); + int64_t* vd64 = (int64_t*)ray_data(vr); + for (int64_t i = 0; i < n; i++) { + kd64[i] = (i * 13) % nk; + vd64[i] = (i * 3) % nv; + kd[i] = (int32_t)kd64[i]; + vd[i] = (int16_t)vd64[i]; + } + k->len = n; v->len = n; kr->len = n; vr->len = n; + + ray_t* r = ray_cd_fused(k, v, n); + TEST_ASSERT_NOT_NULL(r); + test_result_t chk = cdf_verify(r, kd64, vd64, n, nk, nv); + if (chk.status != TEST_PASS) return chk; + ray_release(r); + ray_release(k); ray_release(v); ray_release(kr); ray_release(vr); + + ray_sym_destroy(); + ray_heap_destroy(); + PASS(); +} + /* -------------------------------------------------------------------------- * Test registry * -------------------------------------------------------------------------- */ @@ -2433,5 +2758,11 @@ const test_entry_t group_extra_entries[] = { { "group_extra/hll_count_distinct_approx_pg_stream_types", test_hll_count_distinct_approx_pg_stream_types, NULL, NULL }, { "group_extra/hll_merge_edges", test_hll_merge_edges, NULL, NULL }, { "group_extra/ght_layout_copy_depth_invariance", test_ght_layout_copy_depth_invariance, NULL, NULL }, + { "group_extra/ght_packed_key_excludes_f64", test_ght_packed_key_excludes_f64, NULL, NULL }, + { "group_extra/cd_fused_basic", test_cd_fused_basic, NULL, NULL }, + { "group_extra/cd_fused_interleaved", test_cd_fused_interleaved, NULL, NULL }, + { "group_extra/cd_fused_narrow_types", test_cd_fused_narrow_types, NULL, NULL }, + { "group_extra/cd_fused_skewed", test_cd_fused_skewed, NULL, NULL }, + { "group_extra/cd_fused_self", test_cd_fused_self, NULL, NULL }, { NULL, NULL, NULL, NULL }, }; diff --git a/test/test_heap.c b/test/test_heap.c index 00b6864f..af040c42 100644 --- a/test/test_heap.c +++ b/test/test_heap.c @@ -2209,6 +2209,11 @@ static test_result_t test_order_overflow_guards(void) { * watermark so the crossing is deterministic regardless of machine RAM. */ static test_result_t test_anon_watermark_spill(void) { size_t sz = 40 * 1024 * 1024 - 128; /* order 26 → direct path */ + /* Start from an empty reuse cache: leftover cached blocks from earlier + * tests would (a) inflate the baseline and (b) be drained by the + * second alloc's pressure path, freeing enough headroom for it to stay + * anonymous instead of spilling. */ + ray_heap_direct_cache_drain(); int64_t base = ray_heap_anon_committed(); /* Headroom for exactly one ~40 MB block, not two. */ ray_heap_set_anon_watermark(base + 48 * 1024 * 1024); @@ -2235,6 +2240,10 @@ static test_result_t test_anon_watermark_spill(void) { * blocks or leave the low watermark set for later tests. */ if (a) ray_free(a); if (b) ray_free(b); + /* Freeing a large anon direct block may STASH it in the reuse cache + * (pages stay resident and counted). Drain so the baseline assertion + * below sees the fully-released state. */ + ray_heap_direct_cache_drain(); ray_heap_set_anon_watermark(0); TEST_ASSERT_NOT_NULL(a); diff --git a/test/test_lang.c b/test/test_lang.c index 960591eb..b1272946 100644 --- a/test/test_lang.c +++ b/test/test_lang.c @@ -5224,9 +5224,9 @@ static test_result_t test_dotted_del_cascade(void) { static test_result_t test_select_by_nullable_f64_key(void) { /* Nullable F64 key column: without null-awareness the new hash-based - * first-idx path collided the null group with the 0.0 group — F64 - * null's bit pattern is -0.0, and ray_hash_f64 normalises -0.0 to - * +0.0, so hash(null) == hash(0.0) and the null group got a stale + * first-idx path collided the null group with the 0.0 group — a null + * key is stored as an all-zero key slot, which reads back as +0.0, so + * hash(null) == hash(0.0) and the null group got a stale * first_idx = -1. The indices must point to the actual first row * with that Price value (or first null row). */ ray_eval_str("(set __nv (table [OrderId Price] (list (til 5) [0.0 0Nf 0.0 0Nf 1.0])))"); @@ -5249,6 +5249,53 @@ static test_result_t test_select_by_nullable_f64_key(void) { PASS(); } +/* F64 group keys: -0.0 and +0.0 form ONE group, and the emitted key of that + * group is bitwise +0.0 (issue #407). + * + * ray_hash_f64 normalises -0.0, but group_keys_equal / ght_lanes_equal compare + * the raw 8 key bytes: -0.0 hashed into +0.0's slot chain yet never compared + * equal, so it split off into its own group. Each F64 key builder now reads + * keys through group_key_f64_bits, which canonicalises BEFORE both + * the key store and the hash. + * + * The representative is checked at the BIT level here, not through the REPL: + * ray_format normalises -0.0 on output (format.c clear_neg_zero), so an emitted + * -0.0 would print as "0.0" and no .rfl assertion could see it. Because every + * contributing row stores canonical bits, +0.0 wins no matter which row opened + * the group or which worker got there first. + * + * This test file is built with the debug flags (no -fno-signed-zeros), so the + * -0.0 literal survives to the group kernel and the group count alone fails on + * a pre-fix build; the -0.0-first case additionally uses a runtime multiply so + * the same property is exercised the way release sees it. */ +static test_result_t test_select_by_f64_signed_zero_key(void) { + static const uint64_t POS_ZERO_BITS = UINT64_C(0); + /* -0.0 second (literal), then -0.0 FIRST via a runtime multiply. */ + const char* const setups[2] = { + "(set __sz (table [f v] (list [0.0 -0.0 1.0] [1 2 3])))", + "(set __sz (table [f v] (list (as 'F64 (list (* -1.0 0.0) 0.0 1.0)) [1 2 3])))", + }; + for (int c = 0; c < 2; c++) { + ray_eval_str(setups[c]); + ASSERT_EQ("(count (select {n: (count v) by: {f: f} from: __sz}))", "2"); + /* The zero group is first-seen in both cases (row 0 opens it). */ + ASSERT_EQ("(at (at (select {n: (count v) by: {f: f} from: __sz}) 'n) 0)", "2"); + ray_t* g = ray_eval_str("(at (select {n: (count v) by: {f: f} from: __sz}) 'f)"); + TEST_ASSERT_NOT_NULL(g); + TEST_ASSERT_FALSE(RAY_IS_ERR(g)); + TEST_ASSERT_TRUE(g->type == RAY_F64); + TEST_ASSERT_TRUE(g->len == 2); + uint64_t bits; + memcpy(&bits, &((const double*)ray_data(g))[0], 8); + TEST_ASSERT_FMT(bits == POS_ZERO_BITS, + "merged zero group must emit bitwise +0.0, got %016llx", + (unsigned long long)bits); + ray_release(g); + } + ray_eval_str("(del __sz)"); + PASS(); +} + static test_result_t test_select_by_computed_key_nullable_nonkey(void) { /* Computed key (by: (expr)) takes a third result-build path (lines * around query.c:2120 — ray_group_indices_fn on the computed vector + scatter @@ -9011,6 +9058,7 @@ const test_entry_t lang_entries[] = { { "lang/select_by_f64_perf", test_select_by_f64_perf, lang_setup, lang_teardown }, { "lang/select_by_narrow_int_key", test_select_by_narrow_int_key, lang_setup, lang_teardown }, { "lang/select_by_nullable_f64_key", test_select_by_nullable_f64_key, lang_setup, lang_teardown }, + { "lang/select_by_f64_signed_zero_key", test_select_by_f64_signed_zero_key, lang_setup, lang_teardown }, { "lang/select_by_nullable_i64_key", test_select_by_nullable_i64_key, lang_setup, lang_teardown }, { "lang/select_by_str_nullable_nonkey", test_select_by_str_nullable_nonkey, lang_setup, lang_teardown }, { "lang/select_by_computed_key_nullable_nonkey", test_select_by_computed_key_nullable_nonkey, lang_setup, lang_teardown }, diff --git a/test/test_store.c b/test/test_store.c index 03e37ad1..0903c56b 100644 --- a/test/test_store.c +++ b/test/test_store.c @@ -566,6 +566,55 @@ static test_result_t test_splay_dict_column_roundtrip(void) { PASS(); } +/* ---- test_splay_empty_list_column_roundtrip --------------------------- */ +static test_result_t test_splay_empty_list_column_roundtrip(void) { + (void)!system("rm -rf " TMP_SPLAY_DIR); + + ray_t* ids = ray_vec_new(RAY_I64, 0); + ray_t* who = ray_vec_new(RAY_SYM, 0); + ray_t* sched = ray_list_new(0); + TEST_ASSERT_FALSE(RAY_IS_ERR(ids)); + TEST_ASSERT_FALSE(RAY_IS_ERR(who)); + TEST_ASSERT_FALSE(RAY_IS_ERR(sched)); + + ray_t* tbl = ray_table_new(3); + tbl = ray_table_add_col(tbl, ray_sym_intern("id", 2), ids); + TEST_ASSERT_FALSE(RAY_IS_ERR(tbl)); + tbl = ray_table_add_col(tbl, ray_sym_intern("who", 3), who); + TEST_ASSERT_FALSE(RAY_IS_ERR(tbl)); + tbl = ray_table_add_col(tbl, ray_sym_intern("sched", 5), sched); + TEST_ASSERT_FALSE(RAY_IS_ERR(tbl)); + + const char* sym_path = TMP_SPLAY_DIR "/.sym"; + TEST_ASSERT_EQ_I(ray_splay_save(tbl, TMP_SPLAY_DIR, sym_path), RAY_OK); + ray_t* loaded = ray_read_splayed(TMP_SPLAY_DIR, sym_path); + TEST_ASSERT_NOT_NULL(loaded); + TEST_ASSERT_FALSE(RAY_IS_ERR(loaded)); + TEST_ASSERT_EQ_I(ray_table_ncols(loaded), 3); + TEST_ASSERT_EQ_I(ray_table_nrows(loaded), 0); + + ray_t* loaded_ids = ray_table_get_col(loaded, ray_sym_find("id", 2)); + ray_t* loaded_who = ray_table_get_col(loaded, ray_sym_find("who", 3)); + ray_t* loaded_sched = ray_table_get_col(loaded, ray_sym_find("sched", 5)); + TEST_ASSERT_NOT_NULL(loaded_ids); + TEST_ASSERT_NOT_NULL(loaded_who); + TEST_ASSERT_NOT_NULL(loaded_sched); + TEST_ASSERT_EQ_I(loaded_ids->type, RAY_I64); + TEST_ASSERT_EQ_I(loaded_who->type, RAY_SYM); + TEST_ASSERT_EQ_I(loaded_sched->type, RAY_LIST); + TEST_ASSERT_EQ_I(loaded_ids->len, 0); + TEST_ASSERT_EQ_I(loaded_who->len, 0); + TEST_ASSERT_EQ_I(loaded_sched->len, 0); + + ray_release(loaded); + ray_release(tbl); + ray_release(ids); + ray_release(who); + ray_release(sched); + (void)!system("rm -rf " TMP_SPLAY_DIR); + PASS(); +} + /* A deterministic unsupported column must be rejected before an earlier * column can replace the committed generation. */ static test_result_t test_splay_save_preflight_preserves_generation(void) { @@ -5163,6 +5212,7 @@ const test_entry_t store_entries[] = { { "store/splay_str_column_roundtrip", test_splay_str_column_roundtrip, store_setup, store_teardown }, { "store/splay_short_strv_roundtrip", test_splay_short_strv_roundtrip, store_setup, store_teardown }, { "store/splay_dict_column_roundtrip", test_splay_dict_column_roundtrip, store_setup, store_teardown }, + { "store/splay_empty_list_column_roundtrip", test_splay_empty_list_column_roundtrip, store_setup, store_teardown }, { "store/splay_save_preflight", test_splay_save_preflight_preserves_generation, store_setup, store_teardown }, { "store/parted_nrows", test_parted_nrows, store_setup, store_teardown }, { "store/table_nrows_parted", test_table_nrows_parted, store_setup, store_teardown },