diff --git a/base-convert/Cargo.lock b/base-convert/Cargo.lock index 23c5db4..d0116ec 100644 --- a/base-convert/Cargo.lock +++ b/base-convert/Cargo.lock @@ -151,7 +151,7 @@ dependencies = [ [[package]] name = "base-arch" -version = "0.4.5" +version = "0.4.6" dependencies = [ "anyhow", "base-format", @@ -162,7 +162,7 @@ dependencies = [ [[package]] name = "base-awq" -version = "0.4.5" +version = "0.4.6" dependencies = [ "anyhow", "base-format", @@ -176,7 +176,7 @@ dependencies = [ [[package]] name = "base-convert" -version = "0.4.5" +version = "0.4.6" dependencies = [ "anyhow", "base-arch", @@ -202,7 +202,7 @@ dependencies = [ [[package]] name = "base-format" -version = "0.4.5" +version = "0.4.6" dependencies = [ "anyhow", "bitflags", @@ -218,7 +218,7 @@ dependencies = [ [[package]] name = "base-hub" -version = "0.4.5" +version = "0.4.6" dependencies = [ "anyhow", "base-format", @@ -237,7 +237,7 @@ dependencies = [ [[package]] name = "base-quant" -version = "0.4.5" +version = "0.4.6" dependencies = [ "anyhow", "base-format", @@ -249,7 +249,7 @@ dependencies = [ [[package]] name = "base-readers" -version = "0.4.5" +version = "0.4.6" dependencies = [ "anyhow", "base-format", @@ -263,7 +263,7 @@ dependencies = [ [[package]] name = "base-sign" -version = "0.4.5" +version = "0.4.6" dependencies = [ "anyhow", "base-format", diff --git a/base-convert/Cargo.toml b/base-convert/Cargo.toml index de2d850..b9665ef 100644 --- a/base-convert/Cargo.toml +++ b/base-convert/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.4.5" +version = "0.4.6" edition = "2021" license = "Apache-2.0" repository = "https://github.com/basecompute/baseRT" diff --git a/base-convert/crates/base-convert/src/hub.rs b/base-convert/crates/base-convert/src/hub.rs index 4f004c8..59110da 100644 --- a/base-convert/crates/base-convert/src/hub.rs +++ b/base-convert/crates/base-convert/src/hub.rs @@ -635,6 +635,13 @@ fn serve_ready( let Some(directive) = mtp else { return Ok(path); }; + // Under `auto` a target the catalog pairs with a drafter drafts with that + // drafter, which prepare_speculate_auto installs and superfluid's `auto` + // takes first: its own head is not what the start will use, so a file + // without one is not pulled again for it. + if directive == "auto" && auto_drafter_row(reg, id).is_some() { + return Ok(path); + } let Some(variant) = stale_head_variant(reg, id, &path) else { return Ok(path); }; @@ -723,6 +730,136 @@ fn resolve_model_args(rest: &[String], default_variant: Option<&str>) -> Result< .collect() } +/// Whether a runtime command's args ask for `--speculate auto` (two words or +/// `--speculate=auto`); false when speculation is not asked for, or is asked +/// for by name. +fn speculate_auto_asked(rest: &[String]) -> bool { + for (i, a) in rest.iter().enumerate() { + if a == "--speculate" { + return rest.get(i + 1).map(String::as_str) == Some("auto"); + } + if let Some(v) = a.strip_prefix("--speculate=") { + return v == "auto"; + } + } + false +} + +/// The drafter the catalog has `--speculate auto` serve target `id` with +/// (`speculator_auto` on a sidecar row of a kind `--speculate` takes), on a +/// Metal client: the pairings marked so far were measured there (Qwen3.8-27B +/// and Qwen3.6-35B-A3B with DFlash2 against their in-bundle heads, M5 Pro, +/// 2026-11), so another backend keeps superfluid's own choice until measured. +fn auto_drafter_row<'a>( + reg: &'a MergedRegistry, + id: &str, +) -> Option<&'a base_hub::catalog::CatalogEntry> { + if base_hub::registry::CatalogRegistry::client_backend() != "metal" { + return None; + } + reg.catalog.auto_drafter(id).filter(|row| { + matches!( + row.speculator.as_deref(), + Some("dflash" | "dspark" | "eagle3") + ) + }) +} + +/// The catalog id of the target a served `.base` is: the id its hub-cache +/// sidecar records for a file the catalog installed, else — a bundle kept +/// outside the cache and served by path — the one catalog target that +/// publishes a file of that name. None for a conversion of the user's own. +fn catalog_target_id(reg: &MergedRegistry, path: &Path) -> Option { + if let Some(sidecar) = path + .parent() + .and_then(|dir| cache::read_sidecar(dir).ok().flatten()) + { + return (sidecar.source_kind == "catalog").then_some(sidecar.id); + } + let file = path.file_name()?.to_str()?; + reg.catalog.target_id_of_file(file).map(str::to_string) +} + +/// What `--speculate auto` becomes for the one model `basert serve` starts +/// with, as `(--speculate value over catalog ids, the drafter's id)`: the +/// drafter the catalog pairs with it (auto_drafter_row), at the file that +/// row names. None when the choice stays superfluid's: the bundle carries +/// Google's Gemma 4 head (`gemma4-mtp`, which `auto` takes first and no +/// drafter replaces), the model is not a catalog target, or the catalog +/// pairs it with nothing. +fn speculate_auto_directive(reg: &MergedRegistry, model: &Path) -> Option<(String, String)> { + if crate::serve::embedded_head_kind(model).as_deref() == Some("gemma4-mtp") { + return None; + } + let id = catalog_target_id(reg, model)?; + let row = auto_drafter_row(reg, &id)?; + let kind = row.speculator.as_deref()?; + Some((format!("{kind}:{}:{}", row.id, row.quant), row.id.clone())) +} + +/// `basert serve --speculate auto`, the model already a path: the +/// drafter the catalog pairs with that model (auto_drafter_row) is pulled when +/// it is not installed — the road `--speculate dflash:` takes — and the +/// args go on as they came. `auto` stays superfluid's to settle: it takes that +/// installed drafter first (it reads the same catalog, the cache's +/// `.catalog-cache.json`, even under the `--offline` every start hands it) and +/// keeps its fallbacks, so a copy it cannot use (stale, of another shape, or +/// refused by the engine) gives way to the bundle's own head, then to plain +/// decoding. Naming the file here (`dflash:`) would make it an explicit +/// choice, which superfluid refuses at startup instead. Nothing is pulled for +/// another strategy or none, several models or a `--model-dir` (superfluid's +/// one `--speculate` is not one model's there), a model with nothing paired, +/// or under the user's own `--offline`; a drafter that could not be fetched is +/// said and never stops the start. +fn prepare_speculate_auto(rest: Vec) -> Vec { + if !speculate_auto_asked(&rest) { + return rest; + } + let Ok(reg) = MergedRegistry::load() else { + return rest; + }; + prepare_speculate_auto_with(rest, ®, |token| resolve_hub_model(token, None, None)) +} + +/// prepare_speculate_auto over a given registry, `fetch` resolving (pulling) +/// a drafter's `:` to its installed path. +fn prepare_speculate_auto_with( + rest: Vec, + reg: &MergedRegistry, + fetch: impl FnOnce(&str) -> Result, +) -> Vec { + if !speculate_auto_asked(&rest) || rest.iter().any(|a| a == "--offline") { + return rest; + } + let Ok(crate::serve::Parsed::Serve(o)) = crate::serve::parse(&rest) else { + return rest; + }; + let [model] = o.models.as_slice() else { + return rest; + }; + if o.model_dir.is_some() { + return rest; + } + let Some((directive, drafter)) = speculate_auto_directive(reg, Path::new(model)) else { + return rest; + }; + let Some((_, token)) = speculate_drafter_id(&directive) else { + return rest; + }; + match fetch(token) { + Ok(path) => eprintln!( + "basert serve: --speculate auto: {drafter} is installed ({}); superfluid's auto drafts \ + with it first, the model's own head and plain decoding behind it", + path.display() + ), + Err(e) => eprintln!( + "basert serve: --speculate auto: {drafter} could not be fetched ({e:#}); \ + superfluid's auto settles among what is installed" + ), + } + rest +} + /// `:` for the strategies that take a drafter artifact — /// `dflash:basecompute/Qwen3-4B-DFlash`, `eagle3:org/head:default-q4`. Returns /// None for a path (`dflash:./x.base`, which keeps working unchanged), for a @@ -826,7 +963,7 @@ pub fn dispatch_external(argv: Vec) -> Result<()> { let (variant_flag, rest) = extract_variant_flag(rest)?; let rest = resolve_model_args(&rest, variant_flag.as_deref())?; if cmd == "serve" { - return crate::serve::run(&rest); + return crate::serve::run(&prepare_speculate_auto(rest)); } (vec![format!("basert-{cmd}"), format!("baseRT_{cmd}")], rest) }; @@ -4578,4 +4715,232 @@ mod tests { headless ); } + + #[test] + fn speculate_auto_is_found_in_both_forms() { + let asked = + |v: &[&str]| speculate_auto_asked(&v.iter().map(|s| s.to_string()).collect::>()); + assert!(asked(&["m.base", "--speculate", "auto"])); + assert!(asked(&["--speculate=auto", "m.base"])); + assert!(!asked(&["m.base", "--speculate", "mtp-head"])); + assert!(!asked(&["m.base", "--speculate=dflash:org/d"])); + assert!(!asked(&["m.base", "--speculate"])); + assert!(!asked(&["m.base", "--max-context", "auto"])); + } + + /// `--speculate auto` becomes the drafter the catalog marks for the + /// model (`speculator_auto`), whether the model is the catalog's install + /// or a file of the catalog's name served by path; a Gemma 4 bundle with + /// its own head, a target with nothing marked and a conversion of the + /// user's own keep superfluid's choice. + #[test] + fn auto_takes_the_drafter_the_catalog_marks_for_the_model() { + use base_format::{SpeculatorBundle, FORMAT_VERSION, MAGIC}; + use std::io::Write; + let metal = base_hub::registry::CatalogRegistry::client_backend() == "metal"; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("cache"); + let reg = MergedRegistry::new( + root.clone(), + base_hub::registry::CatalogRegistry::bundled().unwrap(), + ); + // A header-only bundle, with Google's Gemma 4 head or none. + let bundle = |path: &Path, gemma_head: bool| { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let mut h = + crate::embed::fixtures::header("qwen35", serde_json::json!({"hidden_size": 8})); + if gemma_head { + h.speculator = Some(SpeculatorBundle { + arch: "gemma4-mtp".into(), + config: [("kind".to_string(), serde_json::json!("gemma4-mtp"))] + .into_iter() + .collect(), + tensors: vec![], + }); + } + let json = h.to_canonical_json().unwrap(); + let mut f = std::fs::File::create(path).unwrap(); + f.write_all(&MAGIC).unwrap(); + f.write_all(&FORMAT_VERSION.to_le_bytes()).unwrap(); + f.write_all(&(json.len() as u64).to_le_bytes()).unwrap(); + f.write_all(&json).unwrap(); + }; + let directive = |path: &Path| speculate_auto_directive(®, path).map(|(d, _)| d); + let want = |drafter: &str| metal.then(|| format!("dflash:{drafter}:default-q4")); + + // Served by path, outside the cache: matched by the catalog's file name. + let by_path = |name: &str, gemma_head: bool| { + let path = tmp.path().join("models").join(name); + bundle(&path, gemma_head); + path + }; + assert_eq!( + directive(&by_path("Qwen3.8-27B-Q4.base", false)), + want("basecompute/Qwen3.8-27B-DFlash2") + ); + assert_eq!( + directive(&by_path("Qwen3.6-35B-A3B-Q8.base", false)), + want("basecompute/Qwen3.6-35B-A3B-DFlash2") + ); + assert_eq!( + directive(&by_path("Qwen3.6-27B-Q4.base", false)), + None, + "nothing marked for this target" + ); + assert_eq!( + directive(&by_path("gemma-4-26B-A4B-it-Q4.base", true)), + None, + "Gemma 4 keeps its own head" + ); + assert_eq!(directive(&by_path("my-own.base", false)), None); + assert_eq!( + directive(&by_path("Qwen3.8-27B-DFlash2-Q4.base", false)), + None, + "a drafter's file names no target" + ); + + // Installed by a pull: the sidecar names the catalog id. + let installed = |id: &str, variant: &str, source_kind: &str| { + let vdir = cache::variant_dir(&root, id, variant).unwrap(); + let path = cache::base_artifact_path(&vdir); + bundle(&path, false); + cache::write_sidecar( + &vdir, + &HubSidecar { + id: id.into(), + source_kind: source_kind.into(), + hf_repo: id.into(), + source_repo: None, + revision: "main".into(), + variant: variant.into(), + profile: None, + pulled_at: "0".into(), + base_sha256: None, + file: None, + }, + ) + .unwrap(); + path + }; + assert_eq!( + directive(&installed( + "basecompute/Qwen3.8-27B", + "default-q8", + "catalog" + )), + want("basecompute/Qwen3.8-27B-DFlash2") + ); + assert_eq!( + directive(&installed( + "basecompute/Qwen3.8-27B", + "default-f16", + "huggingface" + )), + None, + "a conversion of the user's own" + ); + + // The directive is one `--speculate dflash:` resolves. + if let Some(d) = want("basecompute/Qwen3.8-27B-DFlash2") { + assert_eq!( + speculate_drafter_id(&d), + Some(("dflash", "basecompute/Qwen3.8-27B-DFlash2:default-q4")) + ); + } + + // Under `auto` such a target is not pulled again for an MTP head it + // will not use (a pull here would reach for the network and fail). + if metal { + let id = "basecompute/Qwen3.8-27B"; + let headless = installed(id, "default-q4", "catalog"); + assert!(stale_head_variant(®, id, &headless).is_some()); + assert_eq!( + serve_ready(®, id, headless.clone(), Some("auto")).unwrap(), + headless + ); + } + } + + /// `--speculate auto` reaches superfluid as `auto` when the launcher has + /// prepared the drafter the catalog marks for the model: superfluid's + /// `auto` takes that drafter first and keeps its fallbacks, so a copy it + /// cannot use (here a cached file of another shape) gives way to the + /// model's head, where `dflash:` would fail the start. The marked + /// drafter is fetched once; nothing is fetched for several models, a + /// `--model-dir`, the user's `--offline` or a strategy named by hand, and a + /// failed fetch leaves the args as they came too. + #[test] + fn auto_prepares_the_marked_drafter_and_stays_auto() { + use base_format::{FORMAT_VERSION, MAGIC}; + use std::io::Write; + let metal = base_hub::registry::CatalogRegistry::client_backend() == "metal"; + let tmp = tempfile::tempdir().unwrap(); + let reg = MergedRegistry::new( + tmp.path().join("cache"), + base_hub::registry::CatalogRegistry::bundled().unwrap(), + ); + let bundle = |path: &Path, arch: &str, hidden: u64| { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let h = + crate::embed::fixtures::header(arch, serde_json::json!({"hidden_size": hidden})); + let json = h.to_canonical_json().unwrap(); + let mut f = std::fs::File::create(path).unwrap(); + f.write_all(&MAGIC).unwrap(); + f.write_all(&FORMAT_VERSION.to_le_bytes()).unwrap(); + f.write_all(&(json.len() as u64).to_le_bytes()).unwrap(); + f.write_all(&json).unwrap(); + }; + // The target, served by path under the catalog's file name. + let model = tmp.path().join("models").join("Qwen3.8-27B-Q4.base"); + bundle(&model, "qwen35", 5120); + let model = model.to_string_lossy().into_owned(); + // A cached drafter superfluid cannot use: another architecture and width. + let stale = tmp + .path() + .join("cache/basecompute/Qwen3.8-27B-DFlash2/default-q4/model.base"); + bundle(&stale, "llama", 8); + let args = |v: &[&str]| v.iter().map(|s| s.to_string()).collect::>(); + let run = |rest: Vec, ok: bool| { + let fetched = std::cell::RefCell::new(Vec::::new()); + let out = prepare_speculate_auto_with(rest.clone(), ®, |token| { + fetched.borrow_mut().push(token.to_string()); + if ok { + Ok(stale.clone()) + } else { + Err(anyhow::anyhow!("the Hub is unreachable")) + } + }); + assert_eq!(out, rest, "the args reach superfluid as they came"); + fetched.into_inner() + }; + let marked: Vec = if metal { + vec!["basecompute/Qwen3.8-27B-DFlash2:default-q4".into()] + } else { + vec![] + }; + assert_eq!(run(args(&[&model, "--speculate", "auto"]), true), marked); + assert_eq!(run(args(&["--speculate=auto", &model]), true), marked); + assert_eq!(run(args(&[&model, "--speculate", "auto"]), false), marked); + let none = Vec::::new(); + assert_eq!( + run(args(&[&model, "--speculate", "auto", "--offline"]), true), + none + ); + assert_eq!(run(args(&[&model, "--speculate", "mtp-head"]), true), none); + assert_eq!( + run( + args(&[&model, "--model", &model, "--speculate", "auto"]), + true + ), + none + ); + let dir = tmp.path().join("models").to_string_lossy().into_owned(); + assert_eq!( + run( + args(&[&model, "--model-dir", &dir, "--speculate", "auto"]), + true + ), + none + ); + } } diff --git a/base-convert/crates/base-convert/src/serve.rs b/base-convert/crates/base-convert/src/serve.rs index 0432502..6746dc8 100644 --- a/base-convert/crates/base-convert/src/serve.rs +++ b/base-convert/crates/base-convert/src/serve.rs @@ -71,6 +71,12 @@ Options: throughput). --prefill-budget N Max prompt tokens ingested per engine tick (64-4096). Lower it to trade first-token latency for smoother concurrent decode. + --speculate auto Speculative decoding with what suits the model: the drafter + the catalog pairs with it, pulled on first use (Qwen3.8-27B + and Qwen3.6-35B-A3B: their DFlash2 drafters), else the head + the model's own bundle carries (Gemma 4: Google's MTP head). + A strategy can be named instead: mtp-head, prompt-lookup, + dflash:, off (superfluid's flag). --basert-lib The directory holding libbaseRT to serve with (default: beside basert, or ../lib, or $BASERT_LIB) --print-command Print the superfluid command line this start would run, and exit @@ -417,6 +423,14 @@ pub fn superfluid_args(o: &Options, sessions: &Path) -> Vec { /// A word on the command line that is a model, not a flag's value: a `.base` /// path, or a hub id (`org/name`, with a variant after `:` or not). fn looks_like_model(word: &str) -> bool { + // A drafter named after its strategy (`dflash:`) is + // `--speculate`'s value, whatever it ends with. + if word + .split_once(':') + .is_some_and(|(kind, _)| matches!(kind, "dflash" | "dspark" | "eagle3" | "draft-model")) + { + return false; + } word.ends_with(".base") || (!word.starts_with(['/', '.', '~']) && word.matches('/').count() == 1) } @@ -624,7 +638,7 @@ pub(crate) fn embedded_head_notice( } /// The `kind` of the head a bundle's header carries, if any. -fn embedded_head_kind(path: &Path) -> Option { +pub(crate) fn embedded_head_kind(path: &Path) -> Option { let h = base_format::BaseReader::read_header(path).ok()?; h.speculator? .config @@ -1065,6 +1079,25 @@ mod tests { assert!( !looks_like_model("/tmp/log") && !looks_like_model("auto") && !looks_like_model("512") ); + // A drafter after `--speculate` is that flag's value, by path or id. + let o = serve(&[ + "m.base", + "--speculate", + "dflash:/models/d/model.base", + "--spec-draft-tokens", + "7", + ]); + assert_eq!(o.models, ["m.base"]); + assert_eq!( + o.passthrough, + [ + "--speculate", + "dflash:/models/d/model.base", + "--spec-draft-tokens", + "7" + ] + ); + assert!(!looks_like_model("dspark:org/drafter") && !looks_like_model("eagle3:x.base")); // Handed over last, after the flags every start sets; the user's // engine directory is superfluid's flag, said once. let o = serve(&["m.base", "--basert-lib", "/engine", "--speculate", "auto"]); diff --git a/base-convert/crates/base-hub/catalog.json b/base-convert/crates/base-hub/catalog.json index d5167d8..75ee9c0 100644 --- a/base-convert/crates/base-hub/catalog.json +++ b/base-convert/crates/base-hub/catalog.json @@ -1,6 +1,6 @@ { "schema": 1, - "updated": "2026-11-05", + "updated": "2026-11-06", "models": [ { "id": "basecompute/Llama-3.1-8B-Instruct", @@ -1867,7 +1867,7 @@ "sha256": "44989d3f2129ec5fcd60cb3068b9126247de7577fc6fb7ac216184dbc2c149c3", "speculator": "dflash", "speculator_for": "basecompute/Qwen3.6-35B-A3B", - "speculator_rank": 1 + "speculator_rank": 2 }, { "id": "basecompute/Qwen3.6-35B-A3B-DFlash", @@ -1881,7 +1881,7 @@ "sha256": "56a5bc65f7019d08f231de8f51efb091c066fc9490c026d9188109c51921a485", "speculator": "dflash", "speculator_for": "basecompute/Qwen3.6-35B-A3B", - "speculator_rank": 1 + "speculator_rank": 2 }, { "id": "basecompute/Qwen3.6-35B-A3B-DFlash2", @@ -1895,7 +1895,8 @@ "speculator": "dflash", "source_repo": "incoai/Qwen3.6-35B-A3B-DFlash2", "speculator_for": "basecompute/Qwen3.6-35B-A3B", - "speculator_rank": 1 + "speculator_rank": 1, + "speculator_auto": true }, { "id": "basecompute/Qwen3.6-35B-A3B-DFlash2", @@ -1909,7 +1910,8 @@ "speculator": "dflash", "source_repo": "incoai/Qwen3.6-35B-A3B-DFlash2", "speculator_for": "basecompute/Qwen3.6-35B-A3B", - "speculator_rank": 1 + "speculator_rank": 1, + "speculator_auto": true }, { "id": "basecompute/Qwen3.8-27B-DFlash2", @@ -1923,7 +1925,8 @@ "speculator": "dflash", "source_repo": "z-lab/Qwen3.8-27B-DFlash2", "speculator_for": "basecompute/Qwen3.8-27B", - "speculator_rank": 1 + "speculator_rank": 1, + "speculator_auto": true }, { "id": "basecompute/Qwen3.8-27B-DFlash2", @@ -1937,7 +1940,8 @@ "speculator": "dflash", "source_repo": "z-lab/Qwen3.8-27B-DFlash2", "speculator_for": "basecompute/Qwen3.8-27B", - "speculator_rank": 1 + "speculator_rank": 1, + "speculator_auto": true }, { "id": "basecompute/Qwen3.8-27B-DSpark", diff --git a/base-convert/crates/base-hub/src/catalog.rs b/base-convert/crates/base-hub/src/catalog.rs index 2af565a..fdfe36c 100644 --- a/base-convert/crates/base-hub/src/catalog.rs +++ b/base-convert/crates/base-hub/src/catalog.rs @@ -109,6 +109,15 @@ pub struct CatalogEntry { /// order (dspark, dflash, eagle3). #[serde(default, skip_serializing_if = "Option::is_none")] pub speculator_rank: Option, + /// Sidecar rows only: `basert serve --speculate auto` serves the target + /// (`speculator_for`) with this drafter, ahead of a head the target's + /// own bundle carries, and pulls it when it is not installed. Set where + /// the drafter was measured faster than that head (Qwen3.8-27B and + /// Qwen3.6-35B-A3B with their DFlash2 drafters, Metal). Absent: `auto` + /// is left to superfluid, which takes an in-bundle head first, then an + /// installed drafter by rank. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub speculator_auto: Option, } fn default_file() -> String { diff --git a/base-convert/crates/base-hub/src/gen.rs b/base-convert/crates/base-hub/src/gen.rs index 957f2a7..9984d8d 100644 --- a/base-convert/crates/base-hub/src/gen.rs +++ b/base-convert/crates/base-hub/src/gen.rs @@ -165,6 +165,7 @@ pub fn entry_from_header( }), speculator_for: None, speculator_rank: None, + speculator_auto: None, }) } diff --git a/base-convert/crates/base-hub/src/registry.rs b/base-convert/crates/base-hub/src/registry.rs index 8f0d884..dd6f490 100644 --- a/base-convert/crates/base-hub/src/registry.rs +++ b/base-convert/crates/base-hub/src/registry.rs @@ -329,6 +329,46 @@ impl CatalogRegistry { }) } + /// The drafter `basert serve --speculate auto` serves `target_id` with: + /// the sidecar row marked `speculator_auto` that drafts for that target + /// (`speculator_for`) and runs on this client, lowest rank first, its + /// default-q4 file (the measured one) before any other. None when the + /// catalog marks none. + pub fn auto_drafter(&self, target_id: &str) -> Option<&crate::catalog::CatalogEntry> { + self.catalog + .models + .iter() + .filter(|e| { + e.speculator_auto == Some(true) + && e.speculator_for + .as_deref() + .is_some_and(|t| t.eq_ignore_ascii_case(target_id)) + && Self::backend_ok(e) + }) + .min_by_key(|e| { + ( + e.speculator_rank.unwrap_or(u32::MAX), + u8::from(e.quant != "default-q4"), + ) + }) + } + + /// The id of the one catalog target that publishes a file named `file` + /// (`Qwen3.8-27B-Q4.base`): how a bundle served by path, outside the hub + /// cache, is matched to its catalog row. None when no target row, or + /// rows of more than one id, carry that name; a sidecar's file never + /// names a target. + pub fn target_id_of_file(&self, file: &str) -> Option<&str> { + let mut ids = self + .catalog + .models + .iter() + .filter(|e| e.speculator_for.is_none() && e.file == file) + .map(|e| e.id.as_str()); + let first = ids.next()?; + ids.all(|id| id == first).then_some(first) + } + /// Quant-agnostic, backend-aware resolve. pub fn resolve(&self, id: &str) -> Option { self.resolve_variant(id, None) diff --git a/base-convert/crates/base-hub/src/scan.rs b/base-convert/crates/base-hub/src/scan.rs index e7f836d..8887402 100644 --- a/base-convert/crates/base-hub/src/scan.rs +++ b/base-convert/crates/base-hub/src/scan.rs @@ -230,6 +230,17 @@ fn inherited_speculator_rank(known: &Catalog, repo: &str) -> Option { .and_then(|m| m.speculator_rank) } +/// Whether `--speculate auto` serves the target with this sidecar +/// (`speculator_auto`) is curated beside its rank and carried across the same +/// way. +fn inherited_speculator_auto(known: &Catalog, repo: &str) -> Option { + known + .models + .iter() + .find(|m| m.hf_repo == repo && m.speculator_auto.is_some()) + .and_then(|m| m.speculator_auto) +} + /// Read a published bundle's header without downloading the bundle. /// /// Two reads, because the header's length is in the header: the 16-byte prefix @@ -604,6 +615,7 @@ pub fn scan_org( entry.source_repo = inherited_source_repo(known, repo); entry.speculator_for = inherited_speculator_for(known, repo); entry.speculator_rank = inherited_speculator_rank(known, repo); + entry.speculator_auto = inherited_speculator_auto(known, repo); entry.parts_sha256 = parts_sha256; report.entries.push(entry) } @@ -755,6 +767,7 @@ mod tests { speculator_version: None, speculator_for: None, speculator_rank: None, + speculator_auto: None, }], }; let r = scan_org( @@ -824,6 +837,7 @@ mod tests { speculator_version: None, speculator_for: None, speculator_rank: None, + speculator_auto: None, }, CatalogEntry { id: "basecompute/other".into(), @@ -841,6 +855,7 @@ mod tests { speculator_version: None, speculator_for: None, speculator_rank: None, + speculator_auto: None, }, ], }; @@ -880,6 +895,7 @@ mod tests { speculator_version: None, speculator_for: Some("basecompute/Qwen3-4B".into()), speculator_rank: Some(2), + speculator_auto: Some(true), }, CatalogEntry { id: "basecompute/Qwen3-4B".into(), @@ -897,6 +913,7 @@ mod tests { speculator_version: None, speculator_for: None, speculator_rank: None, + speculator_auto: None, }, ], }; @@ -912,6 +929,14 @@ mod tests { inherited_speculator_rank(&known, "basecompute/Qwen3-4B"), None ); + assert_eq!( + inherited_speculator_auto(&known, "basecompute/Qwen3-4B-DFlash"), + Some(true) + ); + assert_eq!( + inherited_speculator_auto(&known, "basecompute/Qwen3-4B"), + None + ); // A target repo never borrows a sidecar's association. assert_eq!( inherited_speculator_for(&known, "basecompute/Qwen3-4B"), @@ -1231,6 +1256,7 @@ mod tests { speculator_version: None, speculator_for: None, speculator_rank: None, + speculator_auto: None, }], }; let r = scan_org( diff --git a/bindings/node/package.json b/bindings/node/package.json index 14a6634..44ec55b 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@baseRT/node", - "version": "0.4.5", + "version": "0.4.6", "private": true, "description": "Node.js bindings for BaseRT — LLM inference engine for Apple Silicon (Metal)", "main": "dist/index.js", diff --git a/bindings/python/baseRT/__init__.py b/bindings/python/baseRT/__init__.py index 2d7a27a..fa521f0 100644 --- a/bindings/python/baseRT/__init__.py +++ b/bindings/python/baseRT/__init__.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union -__version__ = "0.4.5" +__version__ = "0.4.6" # --------------------------------------------------------------------------- # Library loading diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 604a7f3..896093e 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -6,7 +6,7 @@ setup( name="baseRT", - version="0.4.5", + version="0.4.6", description="Python bindings for the BaseRT LLM inference engine (Apple Silicon / Metal)", long_description=long_description, long_description_content_type="text/markdown", diff --git a/bindings/rust/baseRT-sys/Cargo.toml b/bindings/rust/baseRT-sys/Cargo.toml index f9763d3..0dcbdba 100644 --- a/bindings/rust/baseRT-sys/Cargo.toml +++ b/bindings/rust/baseRT-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT-sys" -version = "0.4.5" +version = "0.4.6" edition = "2021" description = "Raw FFI bindings for the BaseRT LLM inference engine" license = "Apache-2.0" diff --git a/bindings/rust/baseRT/Cargo.toml b/bindings/rust/baseRT/Cargo.toml index 6389ef7..99ef35d 100644 --- a/bindings/rust/baseRT/Cargo.toml +++ b/bindings/rust/baseRT/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT" -version = "0.4.5" +version = "0.4.6" edition = "2021" description = "Safe Rust bindings for the BaseRT LLM inference engine (Apple Silicon)" license = "Apache-2.0" diff --git a/bindings/swift/Sources/CBaseRT/include/baseRT.h b/bindings/swift/Sources/CBaseRT/include/baseRT.h index 1032e4f..7ed4edc 100644 --- a/bindings/swift/Sources/CBaseRT/include/baseRT.h +++ b/bindings/swift/Sources/CBaseRT/include/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 5 +#define BASERT_VERSION_PATCH 6 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. diff --git a/include/baseRT/baseRT.h b/include/baseRT/baseRT.h index 1032e4f..7ed4edc 100644 --- a/include/baseRT/baseRT.h +++ b/include/baseRT/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 5 +#define BASERT_VERSION_PATCH 6 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks.