v2.5.15

.gitignore

@@ -59,3 +59,4 @@ crash-*
 leak-*
 timeout-*
 oom-*
+.superpowers/

Makefile

@@ -50,8 +50,16 @@ RAY_MARCH ?= native
 DEBUG_CFLAGS   = -fPIC $(WARNS) -std=$(STD) -g -O0 -march=$(RAY_MARCH) -DDEBUG \
   -fsanitize=address,undefined -fno-omit-frame-pointer
 RELEASE_CFLAGS = -fPIC $(WARNS) -std=$(STD) -O3 -march=$(RAY_MARCH) \
-  -funroll-loops -fomit-frame-pointer -fno-math-errno \
+  -funroll-loops -fomit-frame-pointer -fno-math-errno -falign-functions=64 \
   -fassociative-math -ffp-contract=fast -fno-signed-zeros -fno-trapping-math
+# -falign-functions=64: start every function on a cache line.  The default
+#   16-byte alignment makes hot leaf functions' cost depend on where unrelated
+#   edits elsewhere in the same translation unit happen to push them —
+#   measured as a reproducible 14% swing on ClickBench q33/q34
+#   (exec_group_sp_dyn_emit, ~45% of those queries) from an edit that path
+#   never executes.  That noise floor is larger than most real optimisations,
+#   so it hides regressions and manufactures phantom ones.  Costs a few KB of
+#   padding.
 # -fassociative-math: license to reorder FP additions/multiplications.
 #   Required for autovectorization of F64 reductions (sum/avg/dot).
 #   Without it, scalar_sum_f64_fn at group.c:1666 is a serial latency
@@ -111,7 +119,7 @@ TSAN_LDFLAGS = -fsanitize=thread
 HARDENED_CFLAGS = -fPIC $(WARNS) -std=$(STD) -O3 -march=$(RAY_MARCH) -g \
   -fno-omit-frame-pointer -DRAY_HARDENED \
   -funroll-loops -fno-math-errno -fassociative-math -ffp-contract=fast \
-  -fno-signed-zeros -fno-trapping-math
+  -fno-signed-zeros -fno-trapping-math -falign-functions=64
 
 CFLAGS  = $(DEBUG_CFLAGS)
 LDFLAGS = $(DEBUG_LDFLAGS)

docs/docs/storage/index.md

@@ -129,6 +129,9 @@ ray_t* trades = ray_read_parted("db", "trades");
 !!! note "Rayfall builtin"
     Use `.db.parted.get` from Rayfall to load partitioned tables: `(.db.parted.get "db" 'trades)`. See the [Rayfall Storage Builtins](#rayfall-storage) section below.
 
+!!! warning "`update` over a partitioned table materializes in memory"
+    `update` on a partitioned table flattens the whole table into memory first — the parted/`MAPCOMMON` columns cannot be mutated in place — so the result is an ordinary in-memory table. It is **not** written back to the store: re-reading the root returns the original values, and the returned table loses its parted / memory-mapped identity.
+
 ### Partition Pruning
 
 The query optimizer recognizes predicates on the `MAPCOMMON` column and eliminates entire partitions from the scan plan. This means a query filtering on a single date in a year of data only touches 1/365th of the files on disk — with zero per-row cost for the pruned partitions.

src/core/platform.c

@@ -39,6 +39,10 @@
 #include <unistd.h>
 #include <pthread.h>
 #include <signal.h>
+#include <stdio.h>
+#if defined(RAY_OS_MACOS)
+#include <sys/sysctl.h>   /* sysctlbyname — hw.physicalcpu */
+#endif
 #include "mem/sys.h"
 
 /* --------------------------------------------------------------------------
@@ -247,6 +251,49 @@ uint32_t ray_thread_count(void) {
     return (n > 0) ? (uint32_t)n : 1;
 }
 
+/* Physical cores (SMT siblings collapsed).  The worker pool's kernels are
+ * memory-bound; two hyperthreads sharing one core's load/store machinery
+ * only add contention (measured: the full ClickBench suite runs ~11%
+ * SLOWER with 32 SMT threads than with the 16 physical cores on a 5950X).
+ * Counts unique (package, core) pairs from sysfs; any read failure falls
+ * back to the logical count so exotic systems keep the old behavior. */
+uint32_t ray_physical_core_count(void) {
+#if defined(RAY_OS_MACOS)
+    int phys = 0;
+    size_t len = sizeof(phys);
+    if (sysctlbyname("hw.physicalcpu", &phys, &len, NULL, 0) == 0 && phys > 0)
+        return (uint32_t)phys;
+    return ray_thread_count();
+#else
+    uint32_t logical = ray_thread_count();
+    /* (package_id << 16) | core_id per cpu; count distinct values. */
+    enum { MAX_IDS = 4096 };
+    uint32_t seen[MAX_IDS];
+    uint32_t n_seen = 0;
+    for (uint32_t cpu = 0; cpu < logical && cpu < MAX_IDS; cpu++) {
+        char path[128];
+        long core = -1, pkg = 0;
+        FILE* f;
+        snprintf(path, sizeof(path),
+                 "/sys/devices/system/cpu/cpu%u/topology/core_id", cpu);
+        f = fopen(path, "r");
+        if (!f) return logical;              /* no topology → fall back */
+        if (fscanf(f, "%ld", &core) != 1) { fclose(f); return logical; }
+        fclose(f);
+        snprintf(path, sizeof(path),
+                 "/sys/devices/system/cpu/cpu%u/topology/physical_package_id",
+                 cpu);
+        f = fopen(path, "r");
+        if (f) { if (fscanf(f, "%ld", &pkg) != 1) pkg = 0; fclose(f); }
+        uint32_t id = ((uint32_t)pkg << 16) | ((uint32_t)core & 0xFFFF);
+        uint32_t j = 0;
+        while (j < n_seen && seen[j] != id) j++;
+        if (j == n_seen) seen[n_seen++] = id;
+    }
+    return n_seen > 0 ? n_seen : logical;
+#endif
+}
+
 /* --------------------------------------------------------------------------
  * Semaphore
  * -------------------------------------------------------------------------- */
@@ -433,6 +480,12 @@ uint32_t ray_thread_count(void) {
     return (uint32_t)si.dwNumberOfProcessors;
 }
 
+/* Windows: no cheap topology read here — fall back to the logical count
+ * (the SMT-aware default sizing is a POSIX-side optimization). */
+uint32_t ray_physical_core_count(void) {
+    return ray_thread_count();
+}
+
 /* --------------------------------------------------------------------------
  * Semaphore
  * -------------------------------------------------------------------------- */

src/core/platform.h

@@ -173,6 +173,9 @@ bool ray_vm_hugepage(void* ptr, size_t size);
 ray_err_t ray_thread_create(ray_thread_t* t, ray_thread_fn fn, void* arg);
 ray_err_t ray_thread_join(ray_thread_t t);
 uint32_t ray_thread_count(void);
+/* Physical cores (SMT siblings collapsed); falls back to the logical
+ * count when topology is unavailable. */
+uint32_t ray_physical_core_count(void);
 
 void ray_parallel_begin(void);
 void ray_parallel_end(void);

src/core/pool.c

@@ -147,7 +147,13 @@ static ray_err_t ray_pool_create_impl(ray_pool_t* pool, uint32_t n_workers,
             long v = strtol(env, NULL, 10);
             n_workers = (v > 0) ? (uint32_t)v : 0;
         } else {
-            uint32_t ncpu = ray_thread_count();
+            /* Physical cores, not SMT threads: the pool's kernels are
+             * memory-bound, and hyperthread pairs sharing one core's
+             * load/store machinery only contend (full ClickBench suite
+             * measured ~11% slower at 32 SMT threads than at the 16
+             * physical cores of a 5950X).  ray_physical_core_count falls
+             * back to the logical count when topology is unreadable. */
+            uint32_t ncpu = ray_physical_core_count();
             n_workers = (ncpu > 1) ? ncpu - 1 : 0;
         }
     }

src/io/csv.c

@@ -1466,14 +1466,17 @@ static int csv_hash_elem_size(int8_t t) {
  * BOOL/U8/I16 where the index would dwarf the column.
  *
  * Returns 1 to attach, 0 to skip. */
-static int csv_should_attach_hash(ray_t* v) {
-    if (!v || RAY_IS_ERR(v)) return 0;
-    int esz = csv_hash_elem_size(v->type);
+/* Payload-level core of the hash-upgrade decision, shared with the
+ * .csv.splayed index builder (ray_splay_build_indexes) so the on-disk
+ * store makes the SAME hash-vs-zone decision the in-memory load does --
+ * a reloaded store must not query slower than a fresh .csv.read. */
+int ray_csv_hash_upgrade_check(int8_t type, int64_t len,
+                               const void* index_payload) {
+    const ray_index_t* ix = (const ray_index_t*)index_payload;
+    int esz = csv_hash_elem_size(type);
     if (esz == 0) return 0;
-    /* Need a chunk_zone we can read for entropy estimation. */
-    if (!(v->attrs & RAY_ATTR_HAS_INDEX) || !v->index) return 0;
-    ray_index_t* ix = ray_index_payload(v->index);
-    if (ix->kind != RAY_IDX_CHUNK_ZONE || ix->u.chunk_zone.is_f64) return 0;
+    if (!ix || ix->kind != RAY_IDX_CHUNK_ZONE || ix->u.chunk_zone.is_f64)
+        return 0;
     uint32_t n_chunks = ix->u.chunk_zone.n_chunks;
     if (n_chunks < 4) return 0;
     const int64_t* mins = (const int64_t*)ray_data(ix->u.chunk_zone.mins);
@@ -1535,7 +1538,7 @@ static int csv_should_attach_hash(ray_t* v) {
      * index dwarfs the data) out of the index set while admitting
      * I32 / I64 numeric IDs.  Done in int64 arithmetic (we cap n
      * to anything that would overflow at the row counts we accept). */
-    int64_t n = v->len;
+    int64_t n = len;
     if (n <= 0) return 0;
     uint64_t cap = 8;
     uint64_t want = (uint64_t)(2 * n);
@@ -1547,6 +1550,13 @@ static int csv_should_attach_hash(ray_t* v) {
     return 1;
 }
 
+static int csv_should_attach_hash(ray_t* v) {
+    if (!v || RAY_IS_ERR(v)) return 0;
+    if (!(v->attrs & RAY_ATTR_HAS_INDEX) || !v->index) return 0;
+    return ray_csv_hash_upgrade_check(v->type, v->len,
+                                      ray_index_payload(v->index));
+}
+
 /* --------------------------------------------------------------------------
  * `INT` schema columns — auto narrowest integer width.
  *
@@ -1953,9 +1963,19 @@ static ray_t* csv_materialize_rows(const char* buf, size_t file_size,
      * After the chunk_zone attaches we re-walk the same columns and
      * upgrade the high-entropy ones to a hash index (the chunk_zone
      * stays as well — it's the entropy signal we just measured).  See
-     * csv_should_attach_hash for the selectivity + memory cap. */
+     * csv_should_attach_hash for the selectivity + memory cap.
+     *
+     * Progress: same treatment as the finalize dispatch above — the
+     * per-column index builds run their own pool dispatches whose row
+     * totals are not n_rows, so letting them drive the progress pump
+     * resets a completed parse to arbitrary fractions (visibly
+     * 100% -> 50% -> ...).  Suppress progress for the whole index
+     * phase; the parse's completed row count stays on screen. */
+    uint32_t idx_qmode = ray_qstats_mode();
+    ray_qstats_set_mode(idx_qmode & ~RAY_QS_PROGRESS);
     for (int c = 0; c < ncols; c++) {
         if (ray_interrupted()) {
+            ray_qstats_set_mode(idx_qmode);
             for (int j = 0; j < ncols; j++) ray_release(col_vecs[j]);
             return ray_error("cancel", "interrupted");
         }
@@ -1964,6 +1984,7 @@ static ray_t* csv_materialize_rows(const char* buf, size_t file_size,
         if (v->len < (1 << 16)) continue;        /* < one chunk, skip */
         ray_t* r = ray_index_attach_chunk_zone(&v, 16);
         if (ray_interrupted()) {
+            ray_qstats_set_mode(idx_qmode);
             for (int j = 0; j < ncols; j++) ray_release(col_vecs[j]);
             return ray_error("cancel", "interrupted");
         }
@@ -1972,6 +1993,7 @@ static ray_t* csv_materialize_rows(const char* buf, size_t file_size,
     }
     for (int c = 0; c < ncols; c++) {
         if (ray_interrupted()) {
+            ray_qstats_set_mode(idx_qmode);
             for (int j = 0; j < ncols; j++) ray_release(col_vecs[j]);
             return ray_error("cancel", "interrupted");
         }
@@ -1984,11 +2006,13 @@ static ray_t* csv_materialize_rows(const char* buf, size_t file_size,
          * anyway, so the chunk_zone is dead weight. */
         ray_t* r = ray_index_attach_hash(&v);
         if (ray_interrupted()) {
+            ray_qstats_set_mode(idx_qmode);
             for (int j = 0; j < ncols; j++) ray_release(col_vecs[j]);
             return ray_error("cancel", "interrupted");
         }
         if (r && !RAY_IS_ERR(r)) col_vecs[c] = v;
     }
+    ray_qstats_set_mode(idx_qmode);
 
     ray_t* tbl = ray_table_new(ncols);
     if (!tbl || RAY_IS_ERR(tbl)) {
@@ -2459,24 +2483,30 @@ ray_t* ray_read_csv_named_opts(const char* path, char delimiter, bool header,
          * matching loop in build_table_from_cols) — unsupported types
          * fall through to the unindexed path inside the consumer.
          * Second pass upgrades high-entropy columns to a hash index;
-         * see csv_should_attach_hash. */
+         * see csv_should_attach_hash.
+         * Progress suppressed for the whole phase (same rationale as the
+         * finalize dispatch): the index builds' pool dispatches would
+         * reset the completed parse progress to arbitrary fractions. */
+        uint32_t idx_qmode = ray_qstats_mode();
+        ray_qstats_set_mode(idx_qmode & ~RAY_QS_PROGRESS);
         for (int c = 0; c < ncols; c++) {
-            if (ray_interrupted()) goto fail_cols_cancel;
+            if (ray_interrupted()) { ray_qstats_set_mode(idx_qmode); goto fail_cols_cancel; }
             ray_t* v = col_vecs[c];
             if (!v || RAY_IS_ERR(v)) continue;
             if (v->len < (1 << 16)) continue;
             ray_t* r = ray_index_attach_chunk_zone(&v, 16);
-            if (ray_interrupted()) goto fail_cols_cancel;
+            if (ray_interrupted()) { ray_qstats_set_mode(idx_qmode); goto fail_cols_cancel; }
             if (r && !RAY_IS_ERR(r)) col_vecs[c] = v;
         }
         for (int c = 0; c < ncols; c++) {
-            if (ray_interrupted()) goto fail_cols_cancel;
+            if (ray_interrupted()) { ray_qstats_set_mode(idx_qmode); goto fail_cols_cancel; }
             ray_t* v = col_vecs[c];
             if (!csv_should_attach_hash(v)) continue;
             ray_t* r = ray_index_attach_hash(&v);
-            if (ray_interrupted()) goto fail_cols_cancel;
+            if (ray_interrupted()) { ray_qstats_set_mode(idx_qmode); goto fail_cols_cancel; }
             if (r && !RAY_IS_ERR(r)) col_vecs[c] = v;
         }
+        ray_qstats_set_mode(idx_qmode);
 
         ray_t* tbl = ray_table_new(ncols);
         if (!tbl || RAY_IS_ERR(tbl)) {

src/io/csv.h

@@ -85,4 +85,10 @@ ray_err_t ray_csv_save_splayed_named_opts(const char* path, char delimiter, bool
                                           const char* dir, int64_t rows_per_chunk);
 ray_err_t ray_write_csv(ray_t* table, const char* path);
 
+/* Hash-vs-chunk-zone index policy for a column, from its chunk-zone entropy
+ * (payload-level; see csv.c).  Shared by the in-memory CSV load and the
+ * .csv.splayed store index builder so both make the same decision. */
+int ray_csv_hash_upgrade_check(int8_t type, int64_t len,
+                               const void* index_payload);
+
 #endif /* RAY_CSV_H */

src/lang/cal.h

@@ -41,20 +41,22 @@ static inline int date_leap_year(int year) {
     return (year % 4 == 0 && year % 100 != 0) || year % 400 == 0;
 }
 
-static inline int32_t date_years_by_days(int yy) {
-    return (int32_t)((int64_t)yy * 365 + yy / 4 - yy / 100 + yy / 400);
+static inline int64_t date_years_by_days(int yy) {
+    return (int64_t)yy * 365 + yy / 4 - yy / 100 + yy / 400;
 }
 
 /* Decode: days-since-epoch → year/month/day */
 static inline void date_to_ymd(int32_t days, int* y, int* m, int* d) {
-    int32_t offset = days + date_years_by_days(RAY_DATE_EPOCH - 1);
+    /* int64 throughout: days near INT32_MAX overflow int32 after the epoch
+     * offset (e.g. (as 'DATE 2147483646)), producing garbage years. */
+    int64_t offset = (int64_t)days + date_years_by_days(RAY_DATE_EPOCH - 1);
     double approx = (double)offset / 365.2425;
     int32_t years = (int32_t)(approx >= 0.0 ? approx + 0.5 : approx - 0.5);
 
     if (date_years_by_days(years) > offset)
         years -= 1;
 
-    int32_t rem = offset - date_years_by_days(years);
+    int64_t rem = offset - date_years_by_days(years);
     int yy = years + 1;
     int leap = date_leap_year(yy);
     int mid = 0;
@@ -68,18 +70,18 @@ static inline void date_to_ymd(int32_t days, int* y, int* m, int* d) {
 
     *y = yy;
     *m = 1 + mid % 12;
-    *d = 1 + rem - (int32_t)MONTHDAYS[leap][mid];
+    *d = 1 + (int)(rem - (int32_t)MONTHDAYS[leap][mid]);
 }
 
 /* Encode: year/month/day → days-since-epoch */
 static inline int32_t ymd_to_date(int year, int month, int day) {
     int yy = (year > 0) ? year - 1 : 0;
-    int32_t ydays = date_years_by_days(yy);
+    int64_t ydays = date_years_by_days(yy);
     int leap = date_leap_year(year);
     int mm = (month > 0) ? month - 1 : 0;
     if (mm > 12) mm = 12;  /* defensive: never index past the 13-wide table */
-    int32_t mdays = (int32_t)MONTHDAYS[leap][mm];
-    return ydays - date_years_by_days(RAY_DATE_EPOCH - 1) + mdays + day - 1;
+    int64_t mdays = (int32_t)MONTHDAYS[leap][mm];
+    return (int32_t)(ydays - date_years_by_days(RAY_DATE_EPOCH - 1) + mdays + day - 1);
 }
 
 #endif /* RAY_CAL_H */