ci: actually build and assert the SIMD kernels, including AVX-512

The unittest target builds every source in a single invocation, which leaves
__AVX2__ and __AVX512F__ undefined: those kernels compile to nothing and the
suite exercises the scalar fallback instead. Every green CI run so far has been
testing the SIMD backends by not testing them - the same mechanism that made
every shipped x86 build scalar.

* unittest-simd compiles per translation unit the way the extension target does,
  so the SIMD kernels are actually linked in. RUNNER wraps the binaries for an
  emulator, EXPECT_BACKEND asserts which tier got installed.

* test/backend.c prints the installed backends and, given an argument, exits
  non-zero unless that is the one that was installed. Without this a silent
  fallback is indistinguishable from a pass: the suite runs, 1447 tests go
  green, and the kernels under test were never reached.

* A new avx512 job runs the suite on those kernels. GitHub's hosted fleet is
  mixed - some runners have AVX-512, some do not, and there is no way to request
  one (actions/runner#1069) - so the job uses the hardware when it is there and
  Intel SDE when it is not, emulating Skylake-X because that is exactly the
  F/BW/VL/DQ set cpu_supports_avx512() requires. SDE is pulled from Intel's own
  download mirror with a pinned SHA-256; a mismatch fails the job rather than
  running an unverified binary.

Verified on linux/amd64: unittest-simd installs the AVX2 backend and passes all
1447 tests (the plain unittest target reports CPU there), and EXPECT_BACKEND
fails loudly on a mismatch. The SDE path could not be exercised locally - Pin
aborts under nested emulation on Apple Silicon - but by construction that path
cannot pass without AVX-512 having run.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

Author: marcobambini

.github/workflows/main.yml

@@ -203,6 +203,69 @@ jobs:
           path: dist/vector.*
           if-no-files-found: error
 
+  avx512:
+    # The AVX-512 kernels are compiled out of every other job: the plain `unittest` target
+    # builds all sources in one invocation, so __AVX512F__ is undefined and the suite
+    # silently exercises the scalar fallback. This job builds them for real and asserts
+    # which backend was installed, so a fallback fails instead of passing quietly.
+    #
+    # GitHub's hosted fleet is mixed - some runners have AVX-512, some do not, and there
+    # is no way to request one (actions/runner#1069). When this runner has it we run the
+    # kernels natively; otherwise we run them under Intel SDE, which emulates the ISA
+    # deterministically. Either way EXPECT_BACKEND makes the job fail if AVX-512 is not
+    # what actually ran.
+    name: avx512 kernels
+    if: ${{ !contains(github.event.head_commit.message, '[auto-update]') }}
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    env:
+      # Intel SDE, from Intel's own download mirror. Bump both together; the job fails on
+      # a checksum mismatch rather than running an unverified binary.
+      SDE_URL: https://downloadmirror.intel.com/924984/sde-external-10.13.1-2026-07-28-lin.tar.xz
+      SDE_SHA256: 94e97d623fec54385686e1e7ba65ebc9941748c05ee451423948334892bf2b50
+    steps:
+      - uses: actions/checkout@v4.2.2
+
+      - name: does this runner have AVX-512?
+        id: cpu
+        run: |
+          grep -m1 '^model name' /proc/cpuinfo
+          missing=""
+          for f in avx512f avx512bw avx512vl avx512dq; do
+            grep -qw "$f" /proc/cpuinfo || missing="$missing $f"
+          done
+          if [ -z "$missing" ]; then
+            echo "native=true" >> "$GITHUB_OUTPUT"
+            echo "::notice title=AVX-512::this runner has AVX-512, running the kernels on hardware"
+          else
+            echo "native=false" >> "$GITHUB_OUTPUT"
+            echo "::notice title=AVX-512::this runner is missing$missing, running the kernels under Intel SDE"
+          fi
+
+      - name: install Intel SDE
+        if: steps.cpu.outputs.native == 'false'
+        run: |
+          curl -fsSL -o /tmp/sde.tar.xz "$SDE_URL"
+          echo "$SDE_SHA256  /tmp/sde.tar.xz" | sha256sum -c -
+          mkdir -p /tmp/sde
+          tar -xJf /tmp/sde.tar.xz -C /tmp/sde --strip-components=1
+          /tmp/sde/sde64 --version | head -2
+
+      - name: run the suite on the AVX-512 kernels
+        run: |
+          if [ "${{ steps.cpu.outputs.native }}" = "true" ]; then
+            make unittest-simd EXPECT_BACKEND=AVX512
+          else
+            make unittest-simd EXPECT_BACKEND=AVX512 RUNNER="/tmp/sde/sde64 -skx --"
+          fi
+
+      - name: run the suite on the AVX2 kernels
+        # Only meaningful where AVX-512 is absent: the runtime check prefers AVX-512
+        # whenever the CPU has it, so asserting AVX2 there would correctly fail. Cheap
+        # extra coverage either way - until now no job built these kernels at all.
+        if: steps.cpu.outputs.native == 'false'
+        run: make unittest-simd EXPECT_BACKEND=AVX2
+
   release:
     runs-on: ubuntu-22.04
     name: release

Makefile

@@ -151,6 +151,36 @@ unittest:
 	$(CC) $(CFLAGS) -DSQLITE_CORE -O2 $(TEST_SRC) -o $(BUILD_DIR)/test_vector -lm -lpthread
 	./$(BUILD_DIR)/test_vector
 
+# The unittest target above builds every source in a single invocation, which leaves
+# __AVX2__ and __AVX512F__ undefined: those kernels compile to nothing and the suite
+# silently exercises the scalar fallback instead. This target compiles per translation
+# unit the way the extension does, so the SIMD backends are actually under test.
+#
+#   make unittest-simd                                    run on whatever this CPU supports
+#   make unittest-simd EXPECT_BACKEND=AVX512              fail unless AVX-512 was installed
+#   make unittest-simd RUNNER="sde64 -spr --"             run under an emulator
+UNITTEST_OBJ = $(patsubst %.c, $(BUILD_DIR)/ut-%.o, $(notdir $(SRC_FILES))) $(BUILD_DIR)/ut-sqlite3.o
+
+$(BUILD_DIR)/ut-distance-avx2.o: ISA_CFLAGS := $(AVX2_CFLAGS)
+$(BUILD_DIR)/ut-distance-avx512.o: ISA_CFLAGS := $(AVX512_CFLAGS)
+
+$(BUILD_DIR)/ut-%.o: %.c
+	$(CC) $(CFLAGS) $(ISA_CFLAGS) -DSQLITE_CORE -O2 -c $< -o $@
+
+$(BUILD_DIR)/backend: test/backend.c $(UNITTEST_OBJ)
+	$(CC) $(CFLAGS) -DSQLITE_CORE -O2 $< $(UNITTEST_OBJ) -o $@ -lm -lpthread
+
+$(BUILD_DIR)/test_vector_simd: test/test_vector.c $(UNITTEST_OBJ)
+	$(CC) $(CFLAGS) -DSQLITE_CORE -O2 $< $(UNITTEST_OBJ) -o $@ -lm -lpthread
+
+# RUNNER wraps both binaries, so an emulator sees the same build the assertion checked
+RUNNER ?=
+EXPECT_BACKEND ?=
+
+unittest-simd: $(BUILD_DIR)/backend $(BUILD_DIR)/test_vector_simd
+	$(RUNNER) ./$(BUILD_DIR)/backend $(EXPECT_BACKEND)
+	$(RUNNER) ./$(BUILD_DIR)/test_vector_simd
+
 # Clean up generated files
 clean:
 	rm -rf $(BUILD_DIR)/* $(DIST_DIR)/* *.gcda *.gcno *.gcov *.sqlite

test/backend.c

@@ -0,0 +1,59 @@
+//
+//  backend.c
+//  sqlitevector
+//
+//  Reports which distance kernels the extension actually installed, and optionally
+//  asserts that it is the expected one. A scan that silently falls back to a lower tier
+//  passes every test, so CI needs a way to tell "ran on AVX-512" from "meant to".
+//
+//    ./backend            print the installed backends
+//    ./backend AVX512     print them, and exit non-zero unless the distance backend is AVX512
+//
+
+#include <stdio.h>
+#include <string.h>
+
+#include "sqlite3.h"
+
+extern int sqlite3_vector_init (sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi);
+
+int main (int argc, char **argv) {
+    sqlite3 *db = NULL;
+    if (sqlite3_open(":memory:", &db) != SQLITE_OK) {
+        fprintf(stderr, "unable to open an in-memory database\n");
+        return 2;
+    }
+
+    int rc = sqlite3_vector_init(db, NULL, NULL);
+    if (rc != SQLITE_OK) {
+        fprintf(stderr, "sqlite3_vector_init failed (%d)\n", rc);
+        sqlite3_close(db);
+        return 2;
+    }
+
+    sqlite3_stmt *stmt = NULL;
+    if (sqlite3_prepare_v2(db, "SELECT vector_backend(), vector_turboquant_backend();", -1, &stmt, NULL) != SQLITE_OK ||
+        sqlite3_step(stmt) != SQLITE_ROW) {
+        fprintf(stderr, "unable to read the installed backends: %s\n", sqlite3_errmsg(db));
+        sqlite3_finalize(stmt);
+        sqlite3_close(db);
+        return 2;
+    }
+
+    const char *distance = (const char *)sqlite3_column_text(stmt, 0);
+    const char *turbo = (const char *)sqlite3_column_text(stmt, 1);
+    if (!distance) distance = "?";
+    if (!turbo) turbo = "?";
+    printf("distance backend:   %s\n", distance);
+    printf("turboquant backend: %s\n", turbo);
+
+    int result = 0;
+    if (argc > 1) {
+        result = (strcmp(distance, argv[1]) == 0) ? 0 : 1;
+        if (result) fprintf(stderr, "expected the %s backend, but %s was installed\n", argv[1], distance);
+    }
+
+    sqlite3_finalize(stmt);
+    sqlite3_close(db);
+    return result;
+}