From 763a808dbef3786697e87d07dbc7090d3a7b9a2a Mon Sep 17 00:00:00 2001 From: IsuraManchanayake Date: Wed, 11 Nov 2026 01:10:57 +1100 Subject: [PATCH] sync: open-core source for 0.4.7 --- base-convert/Cargo.lock | 16 +++++------ base-convert/Cargo.toml | 2 +- bindings/node/package.json | 2 +- bindings/node/src/index.ts | 6 ++++ bindings/python/baseRT/__init__.py | 2 +- bindings/python/setup.py | 2 +- bindings/rust/baseRT-sys/Cargo.toml | 2 +- bindings/rust/baseRT-sys/src/lib.rs | 2 ++ bindings/rust/baseRT/Cargo.toml | 2 +- .../swift/Sources/BaseRT/BaseRTEngine.swift | 6 ++++ .../swift/Sources/CBaseRT/include/baseRT.h | 28 ++++++++++++++++++- include/baseRT/baseRT.h | 28 ++++++++++++++++++- 12 files changed, 82 insertions(+), 16 deletions(-) diff --git a/base-convert/Cargo.lock b/base-convert/Cargo.lock index d0116ec..4c88c50 100644 --- a/base-convert/Cargo.lock +++ b/base-convert/Cargo.lock @@ -151,7 +151,7 @@ dependencies = [ [[package]] name = "base-arch" -version = "0.4.6" +version = "0.4.7" dependencies = [ "anyhow", "base-format", @@ -162,7 +162,7 @@ dependencies = [ [[package]] name = "base-awq" -version = "0.4.6" +version = "0.4.7" dependencies = [ "anyhow", "base-format", @@ -176,7 +176,7 @@ dependencies = [ [[package]] name = "base-convert" -version = "0.4.6" +version = "0.4.7" dependencies = [ "anyhow", "base-arch", @@ -202,7 +202,7 @@ dependencies = [ [[package]] name = "base-format" -version = "0.4.6" +version = "0.4.7" dependencies = [ "anyhow", "bitflags", @@ -218,7 +218,7 @@ dependencies = [ [[package]] name = "base-hub" -version = "0.4.6" +version = "0.4.7" dependencies = [ "anyhow", "base-format", @@ -237,7 +237,7 @@ dependencies = [ [[package]] name = "base-quant" -version = "0.4.6" +version = "0.4.7" dependencies = [ "anyhow", "base-format", @@ -249,7 +249,7 @@ dependencies = [ [[package]] name = "base-readers" -version = "0.4.6" +version = "0.4.7" dependencies = [ "anyhow", "base-format", @@ -263,7 +263,7 @@ dependencies = [ [[package]] name = "base-sign" -version = "0.4.6" +version = "0.4.7" dependencies = [ "anyhow", "base-format", diff --git a/base-convert/Cargo.toml b/base-convert/Cargo.toml index b9665ef..1dd9f5b 100644 --- a/base-convert/Cargo.toml +++ b/base-convert/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.4.6" +version = "0.4.7" edition = "2021" license = "Apache-2.0" repository = "https://github.com/basecompute/baseRT" diff --git a/bindings/node/package.json b/bindings/node/package.json index 44ec55b..1b53b17 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@baseRT/node", - "version": "0.4.6", + "version": "0.4.7", "private": true, "description": "Node.js bindings for BaseRT — LLM inference engine for Apple Silicon (Metal)", "main": "dist/index.js", diff --git a/bindings/node/src/index.ts b/bindings/node/src/index.ts index 7b0a7d1..80986ee 100644 --- a/bindings/node/src/index.ts +++ b/bindings/node/src/index.ts @@ -410,6 +410,7 @@ interface BaseRTLib { baseRT_set_max_batch_size: (n: number) => void; baseRT_set_prefill_chunk: (n: number) => void; baseRT_set_paged_weights: (mode: number) => void; + baseRT_set_neural_engine: (enable: number) => void; baseRT_set_baked_decode: (enable: number) => void; baseRT_set_gpu_wait_timeout_ms: (ms: number) => void; baseRT_encode: ( @@ -559,6 +560,7 @@ function lib(): BaseRTLib { baseRT_set_max_batch_size: k.func("void baseRT_set_max_batch_size(int)"), baseRT_set_prefill_chunk: k.func("void baseRT_set_prefill_chunk(int)"), baseRT_set_paged_weights: k.func("void baseRT_set_paged_weights(int)"), + baseRT_set_neural_engine: k.func("void baseRT_set_neural_engine(int)"), baseRT_set_baked_decode: k.func("void baseRT_set_baked_decode(int)"), baseRT_set_gpu_wait_timeout_ms: k.func("void baseRT_set_gpu_wait_timeout_ms(double)"), baseRT_encode: k.func( @@ -699,6 +701,10 @@ export const Engine = { setPagedWeights(mode: number): void { lib().baseRT_set_paged_weights(mode); }, + /** The prefill's Neural Engine split: true=on (default), false=GPU alone. */ + setNeuralEngine(enable: boolean): void { + lib().baseRT_set_neural_engine(enable ? 1 : 0); + }, /** Toggle the baked-decode fast path: true=on (default), false=force live. */ setBakedDecode(enable: boolean): void { lib().baseRT_set_baked_decode(enable ? 1 : 0); diff --git a/bindings/python/baseRT/__init__.py b/bindings/python/baseRT/__init__.py index fa521f0..4e9664c 100644 --- a/bindings/python/baseRT/__init__.py +++ b/bindings/python/baseRT/__init__.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union -__version__ = "0.4.6" +__version__ = "0.4.7" # --------------------------------------------------------------------------- # Library loading diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 896093e..2b5264f 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -6,7 +6,7 @@ setup( name="baseRT", - version="0.4.6", + version="0.4.7", description="Python bindings for the BaseRT LLM inference engine (Apple Silicon / Metal)", long_description=long_description, long_description_content_type="text/markdown", diff --git a/bindings/rust/baseRT-sys/Cargo.toml b/bindings/rust/baseRT-sys/Cargo.toml index 0dcbdba..d5c5b2f 100644 --- a/bindings/rust/baseRT-sys/Cargo.toml +++ b/bindings/rust/baseRT-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT-sys" -version = "0.4.6" +version = "0.4.7" edition = "2021" description = "Raw FFI bindings for the BaseRT LLM inference engine" license = "Apache-2.0" diff --git a/bindings/rust/baseRT-sys/src/lib.rs b/bindings/rust/baseRT-sys/src/lib.rs index b082f71..1d6ca19 100644 --- a/bindings/rust/baseRT-sys/src/lib.rs +++ b/bindings/rust/baseRT-sys/src/lib.rs @@ -637,6 +637,8 @@ extern "C" { pub fn baseRT_set_max_batch_size(n: c_int); pub fn baseRT_set_prefix_cache(enable: c_int); pub fn baseRT_set_prefill_chunk(n: c_int); + pub fn baseRT_set_neural_engine(enable: c_int); + pub fn baseRT_prepare_neural_engine(model_path: *const c_char, max_context: c_int) -> c_int; pub fn baseRT_set_kv_bits(bits: c_int); pub fn baseRT_set_verbose(on: c_int); diff --git a/bindings/rust/baseRT/Cargo.toml b/bindings/rust/baseRT/Cargo.toml index 99ef35d..c1c6658 100644 --- a/bindings/rust/baseRT/Cargo.toml +++ b/bindings/rust/baseRT/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT" -version = "0.4.6" +version = "0.4.7" edition = "2021" description = "Safe Rust bindings for the BaseRT LLM inference engine (Apple Silicon)" license = "Apache-2.0" diff --git a/bindings/swift/Sources/BaseRT/BaseRTEngine.swift b/bindings/swift/Sources/BaseRT/BaseRTEngine.swift index 752145b..e97a369 100644 --- a/bindings/swift/Sources/BaseRT/BaseRTEngine.swift +++ b/bindings/swift/Sources/BaseRT/BaseRTEngine.swift @@ -23,6 +23,12 @@ public enum BaseRTEngine { baseRT_set_paged_kv(enable ? 1 : 0) } + /// The prefill's Neural Engine split for subsequent loads: on by default; + /// false runs every prefill on the GPU alone. + public static func setNeuralEngine(_ enable: Bool) { + baseRT_set_neural_engine(enable ? 1 : 0) + } + /// Max in-flight batch size for batched decode (sizes the logits scratch). public static func setMaxBatchSize(_ n: Int32) { baseRT_set_max_batch_size(n) diff --git a/bindings/swift/Sources/CBaseRT/include/baseRT.h b/bindings/swift/Sources/CBaseRT/include/baseRT.h index 7ed4edc..ec18d9d 100644 --- a/bindings/swift/Sources/CBaseRT/include/baseRT.h +++ b/bindings/swift/Sources/CBaseRT/include/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 6 +#define BASERT_VERSION_PATCH 7 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. @@ -253,6 +253,32 @@ void baseRT_set_prefill_chunk(int n); /// Process-wide; read at load. Set before baseRT_load_model. void baseRT_set_paged_weights(int mode); +/// The prefill's Neural Engine split (Apple silicon, Metal backend). +/// enable = 1 → on (default): a prefill chunk's dense FFN / projections are +/// split between the GPU and the Neural Engine where that is faster for +/// the chip and the model +/// enable = 0 → off: the GPU runs every prefill alone (bit-for-bit the +/// GPU path; the split's int8 part differs from it by ~2.5% RMS) +/// Process-wide; read at load. Set before baseRT_load_model. +void baseRT_set_neural_engine(int enable); + +/// Compiles ahead the Neural Engine programs the prefill's split would serve +/// `model_path` with on this Mac, from the bundle's header alone (no weights +/// are read), so that the model's first load does not compile them. A load +/// whose programs are not compiled yet compiles them in the background (tens +/// of seconds) while the GPU runs the prefill alone, and the split takes +/// over from the next prefill chunk once they are ready. The ANE service +/// keeps compilations per calling executable (by its code-signing +/// identifier) until macOS updates or clears its caches: call this from the +/// executable that will load the model. `max_context`: the context the model +/// will be served at, as passed to baseRT_load_model (0: the context a load +/// given 0 picks), which can size the prefill chunk the programs are built +/// for. Returns the programs compiled (0: +/// compiled already), -1 when the split takes nothing of this model on this +/// machine (another backend or chip, a format it does not split) and -2 +/// when a compile failed; baseRT_get_error() says why. +int baseRT_prepare_neural_engine(const char *model_path, int max_context); + /// Toggle the baked-decode fast path (DispatchTable replay). /// enable = 1 → on (default): replay the baked table when eligible /// enable = 0 → force the live fused decode path (baked-vs-live A/B) diff --git a/include/baseRT/baseRT.h b/include/baseRT/baseRT.h index 7ed4edc..ec18d9d 100644 --- a/include/baseRT/baseRT.h +++ b/include/baseRT/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 6 +#define BASERT_VERSION_PATCH 7 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. @@ -253,6 +253,32 @@ void baseRT_set_prefill_chunk(int n); /// Process-wide; read at load. Set before baseRT_load_model. void baseRT_set_paged_weights(int mode); +/// The prefill's Neural Engine split (Apple silicon, Metal backend). +/// enable = 1 → on (default): a prefill chunk's dense FFN / projections are +/// split between the GPU and the Neural Engine where that is faster for +/// the chip and the model +/// enable = 0 → off: the GPU runs every prefill alone (bit-for-bit the +/// GPU path; the split's int8 part differs from it by ~2.5% RMS) +/// Process-wide; read at load. Set before baseRT_load_model. +void baseRT_set_neural_engine(int enable); + +/// Compiles ahead the Neural Engine programs the prefill's split would serve +/// `model_path` with on this Mac, from the bundle's header alone (no weights +/// are read), so that the model's first load does not compile them. A load +/// whose programs are not compiled yet compiles them in the background (tens +/// of seconds) while the GPU runs the prefill alone, and the split takes +/// over from the next prefill chunk once they are ready. The ANE service +/// keeps compilations per calling executable (by its code-signing +/// identifier) until macOS updates or clears its caches: call this from the +/// executable that will load the model. `max_context`: the context the model +/// will be served at, as passed to baseRT_load_model (0: the context a load +/// given 0 picks), which can size the prefill chunk the programs are built +/// for. Returns the programs compiled (0: +/// compiled already), -1 when the split takes nothing of this model on this +/// machine (another backend or chip, a format it does not split) and -2 +/// when a compile failed; baseRT_get_error() says why. +int baseRT_prepare_neural_engine(const char *model_path, int max_context); + /// Toggle the baked-decode fast path (DispatchTable replay). /// enable = 1 → on (default): replay the baked table when eligible /// enable = 0 → force the live fused decode path (baked-vs-live A/B)