sync: open-core source for 0.4.7

base-convert/Cargo.toml

@@ -12,7 +12,7 @@ members = [
 ]
 
 [workspace.package]
-version = "0.4.6"
+version = "0.4.7"
 edition = "2021"
 license = "Apache-2.0"
 repository = "https://github.com/basecompute/baseRT"

bindings/node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@baseRT/node",
-  "version": "0.4.6",
+  "version": "0.4.7",
   "private": true,
   "description": "Node.js bindings for BaseRT — LLM inference engine for Apple Silicon (Metal)",
   "main": "dist/index.js",

bindings/node/src/index.ts

@@ -410,6 +410,7 @@ interface BaseRTLib {
   baseRT_set_max_batch_size: (n: number) => void;
   baseRT_set_prefill_chunk: (n: number) => void;
   baseRT_set_paged_weights: (mode: number) => void;
+  baseRT_set_neural_engine: (enable: number) => void;
   baseRT_set_baked_decode: (enable: number) => void;
   baseRT_set_gpu_wait_timeout_ms: (ms: number) => void;
   baseRT_encode: (
@@ -559,6 +560,7 @@ function lib(): BaseRTLib {
     baseRT_set_max_batch_size: k.func("void baseRT_set_max_batch_size(int)"),
     baseRT_set_prefill_chunk: k.func("void baseRT_set_prefill_chunk(int)"),
     baseRT_set_paged_weights: k.func("void baseRT_set_paged_weights(int)"),
+    baseRT_set_neural_engine: k.func("void baseRT_set_neural_engine(int)"),
     baseRT_set_baked_decode: k.func("void baseRT_set_baked_decode(int)"),
     baseRT_set_gpu_wait_timeout_ms: k.func("void baseRT_set_gpu_wait_timeout_ms(double)"),
     baseRT_encode: k.func(
@@ -699,6 +701,10 @@ export const Engine = {
   setPagedWeights(mode: number): void {
     lib().baseRT_set_paged_weights(mode);
   },
+  /** The prefill's Neural Engine split: true=on (default), false=GPU alone. */
+  setNeuralEngine(enable: boolean): void {
+    lib().baseRT_set_neural_engine(enable ? 1 : 0);
+  },
   /** Toggle the baked-decode fast path: true=on (default), false=force live. */
   setBakedDecode(enable: boolean): void {
     lib().baseRT_set_baked_decode(enable ? 1 : 0);

bindings/python/baseRT/__init__.py

@@ -21,7 +21,7 @@
 from pathlib import Path
 from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
 
-__version__ = "0.4.6"
+__version__ = "0.4.7"
 
 # ---------------------------------------------------------------------------
 # Library loading

bindings/python/setup.py

@@ -6,7 +6,7 @@
 
 setup(
     name="baseRT",
-    version="0.4.6",
+    version="0.4.7",
     description="Python bindings for the BaseRT LLM inference engine (Apple Silicon / Metal)",
     long_description=long_description,
     long_description_content_type="text/markdown",

bindings/rust/baseRT-sys/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "baseRT-sys"
-version = "0.4.6"
+version = "0.4.7"
 edition = "2021"
 description = "Raw FFI bindings for the BaseRT LLM inference engine"
 license = "Apache-2.0"

bindings/rust/baseRT-sys/src/lib.rs

@@ -637,6 +637,8 @@ extern "C" {
     pub fn baseRT_set_max_batch_size(n: c_int);
     pub fn baseRT_set_prefix_cache(enable: c_int);
     pub fn baseRT_set_prefill_chunk(n: c_int);
+    pub fn baseRT_set_neural_engine(enable: c_int);
+    pub fn baseRT_prepare_neural_engine(model_path: *const c_char, max_context: c_int) -> c_int;
     pub fn baseRT_set_kv_bits(bits: c_int);
     pub fn baseRT_set_verbose(on: c_int);
 

bindings/rust/baseRT/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "baseRT"
-version = "0.4.6"
+version = "0.4.7"
 edition = "2021"
 description = "Safe Rust bindings for the BaseRT LLM inference engine (Apple Silicon)"
 license = "Apache-2.0"

bindings/swift/Sources/BaseRT/BaseRTEngine.swift

@@ -23,6 +23,12 @@ public enum BaseRTEngine {
         baseRT_set_paged_kv(enable ? 1 : 0)
     }
 
+    /// The prefill's Neural Engine split for subsequent loads: on by default;
+    /// false runs every prefill on the GPU alone.
+    public static func setNeuralEngine(_ enable: Bool) {
+        baseRT_set_neural_engine(enable ? 1 : 0)
+    }
+
     /// Max in-flight batch size for batched decode (sizes the logits scratch).
     public static func setMaxBatchSize(_ n: Int32) {
         baseRT_set_max_batch_size(n)

bindings/swift/Sources/CBaseRT/include/baseRT.h

@@ -64,7 +64,7 @@ extern "C" {
 
 #define BASERT_VERSION_MAJOR 0
 #define BASERT_VERSION_MINOR 4
-#define BASERT_VERSION_PATCH 6
+#define BASERT_VERSION_PATCH 7
 
 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`.
 /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks.
@@ -253,6 +253,32 @@ void baseRT_set_prefill_chunk(int n);
 /// Process-wide; read at load. Set before baseRT_load_model.
 void baseRT_set_paged_weights(int mode);
 
+/// The prefill's Neural Engine split (Apple silicon, Metal backend).
+///   enable = 1 → on (default): a prefill chunk's dense FFN / projections are
+///     split between the GPU and the Neural Engine where that is faster for
+///     the chip and the model
+///   enable = 0 → off: the GPU runs every prefill alone (bit-for-bit the
+///     GPU path; the split's int8 part differs from it by ~2.5% RMS)
+/// Process-wide; read at load. Set before baseRT_load_model.
+void baseRT_set_neural_engine(int enable);
+
+/// Compiles ahead the Neural Engine programs the prefill's split would serve
+/// `model_path` with on this Mac, from the bundle's header alone (no weights
+/// are read), so that the model's first load does not compile them. A load
+/// whose programs are not compiled yet compiles them in the background (tens
+/// of seconds) while the GPU runs the prefill alone, and the split takes
+/// over from the next prefill chunk once they are ready. The ANE service
+/// keeps compilations per calling executable (by its code-signing
+/// identifier) until macOS updates or clears its caches: call this from the
+/// executable that will load the model. `max_context`: the context the model
+/// will be served at, as passed to baseRT_load_model (0: the context a load
+/// given 0 picks), which can size the prefill chunk the programs are built
+/// for. Returns the programs compiled (0:
+/// compiled already), -1 when the split takes nothing of this model on this
+/// machine (another backend or chip, a format it does not split) and -2
+/// when a compile failed; baseRT_get_error() says why.
+int baseRT_prepare_neural_engine(const char *model_path, int max_context);
+
 /// Toggle the baked-decode fast path (DispatchTable replay).
 ///   enable = 1 → on (default): replay the baked table when eligible
 ///   enable = 0 → force the live fused decode path (baked-vs-live A/B)

include/baseRT/baseRT.h

@@ -64,7 +64,7 @@ extern "C" {
 
 #define BASERT_VERSION_MAJOR 0
 #define BASERT_VERSION_MINOR 4
-#define BASERT_VERSION_PATCH 6
+#define BASERT_VERSION_PATCH 7
 
 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`.
 /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks.
@@ -253,6 +253,32 @@ void baseRT_set_prefill_chunk(int n);
 /// Process-wide; read at load. Set before baseRT_load_model.
 void baseRT_set_paged_weights(int mode);
 
+/// The prefill's Neural Engine split (Apple silicon, Metal backend).
+///   enable = 1 → on (default): a prefill chunk's dense FFN / projections are
+///     split between the GPU and the Neural Engine where that is faster for
+///     the chip and the model
+///   enable = 0 → off: the GPU runs every prefill alone (bit-for-bit the
+///     GPU path; the split's int8 part differs from it by ~2.5% RMS)
+/// Process-wide; read at load. Set before baseRT_load_model.
+void baseRT_set_neural_engine(int enable);
+
+/// Compiles ahead the Neural Engine programs the prefill's split would serve
+/// `model_path` with on this Mac, from the bundle's header alone (no weights
+/// are read), so that the model's first load does not compile them. A load
+/// whose programs are not compiled yet compiles them in the background (tens
+/// of seconds) while the GPU runs the prefill alone, and the split takes
+/// over from the next prefill chunk once they are ready. The ANE service
+/// keeps compilations per calling executable (by its code-signing
+/// identifier) until macOS updates or clears its caches: call this from the
+/// executable that will load the model. `max_context`: the context the model
+/// will be served at, as passed to baseRT_load_model (0: the context a load
+/// given 0 picks), which can size the prefill chunk the programs are built
+/// for. Returns the programs compiled (0:
+/// compiled already), -1 when the split takes nothing of this model on this
+/// machine (another backend or chip, a format it does not split) and -2
+/// when a compile failed; baseRT_get_error() says why.
+int baseRT_prepare_neural_engine(const char *model_path, int max_context);
+
 /// Toggle the baked-decode fast path (DispatchTable replay).
 ///   enable = 1 → on (default): replay the baked table when eligible
 ///   enable = 0 → force the live fused decode path (baked-vs-live A/B)