From 56e792537a06046dcd73af832ee836f6078b53ec Mon Sep 17 00:00:00 2001 From: baseRT sync Date: Fri, 6 Nov 2026 00:42:08 +1100 Subject: [PATCH] sync: open-core source for 0.4.5 Generated by tools/sync_public.py from the internal tree at 1b0e9f344. Only allow-listed paths are included; tuned quant profiles and internal tooling are excluded by rule. --- base-convert/Cargo.lock | 16 +- base-convert/Cargo.toml | 2 +- base-convert/README.md | 6 + base-convert/crates/base-convert/src/embed.rs | 222 ++++++- base-convert/crates/base-convert/src/hub.rs | 629 ++++++++++++++++-- base-convert/crates/base-convert/src/main.rs | 27 +- bindings/node/package.json | 2 +- bindings/python/baseRT/__init__.py | 2 +- bindings/python/setup.py | 2 +- bindings/rust/baseRT-sys/Cargo.toml | 2 +- bindings/rust/baseRT/Cargo.toml | 2 +- .../swift/Sources/CBaseRT/include/baseRT.h | 2 +- include/baseRT/baseRT.h | 2 +- 13 files changed, 796 insertions(+), 120 deletions(-) diff --git a/base-convert/Cargo.lock b/base-convert/Cargo.lock index 37a2637..23c5db4 100644 --- a/base-convert/Cargo.lock +++ b/base-convert/Cargo.lock @@ -151,7 +151,7 @@ dependencies = [ [[package]] name = "base-arch" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "base-format", @@ -162,7 +162,7 @@ dependencies = [ [[package]] name = "base-awq" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "base-format", @@ -176,7 +176,7 @@ dependencies = [ [[package]] name = "base-convert" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "base-arch", @@ -202,7 +202,7 @@ dependencies = [ [[package]] name = "base-format" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "bitflags", @@ -218,7 +218,7 @@ dependencies = [ [[package]] name = "base-hub" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "base-format", @@ -237,7 +237,7 @@ dependencies = [ [[package]] name = "base-quant" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "base-format", @@ -249,7 +249,7 @@ dependencies = [ [[package]] name = "base-readers" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "base-format", @@ -263,7 +263,7 @@ dependencies = [ [[package]] name = "base-sign" -version = "0.4.4" +version = "0.4.5" dependencies = [ "anyhow", "base-format", diff --git a/base-convert/Cargo.toml b/base-convert/Cargo.toml index c322b88..de2d850 100644 --- a/base-convert/Cargo.toml +++ b/base-convert/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.4.4" +version = "0.4.5" edition = "2021" license = "Apache-2.0" repository = "https://github.com/basecompute/baseRT" diff --git a/base-convert/README.md b/base-convert/README.md index 27416f6..55e52a5 100644 --- a/base-convert/README.md +++ b/base-convert/README.md @@ -47,6 +47,12 @@ basert pull Qwen/Qwen3-0.6B --target base-q8 basert pull basecompute/:default-q8 basert pull Qwen/Qwen3-0.6B:q8 +# A release that comes in sizes (the BaseQuant repos' -XS/-Small/-Medium/ +# -Large/-XL files): its Medium by default, another by size or file name: +basert pull basecompute/Qwen3.8-27B-BaseQuant-V1 +basert pull basecompute/Qwen3.8-27B-BaseQuant-V1:small +basert pull basecompute/Qwen3.8-27B-BaseQuant-V1 --file Qwen3.8-27B-BaseQuant-V1-XL.base + basert pull --dry-run # resolve + print the plan, download nothing basert list # installed models (table; --json for JSON) basert list --remote # also show catalog models not yet installed diff --git a/base-convert/crates/base-convert/src/embed.rs b/base-convert/crates/base-convert/src/embed.rs index 70ff056..f058b11 100644 --- a/base-convert/crates/base-convert/src/embed.rs +++ b/base-convert/crates/base-convert/src/embed.rs @@ -7,10 +7,17 @@ //! //! Byte-level: the target's weights blob is copied verbatim, so every main //! and mmproj tensor keeps its bytes, offsets and checksums; the head's -//! tensors are appended behind it, placed by the target's alignment rules -//! exactly as the writer would place them; the header gains the section and +//! tensors are appended behind it, each placed as the writer places a +//! tensor of its region by the stricter of the two files' alignment rules, +//! the first of them on a GPU page (the runtime wraps the section as one +//! buffer from its first entry); the header gains the section and //! `HAS_SPECULATOR`. Nothing is re-quantized, and a reconverted target is -//! not needed: the catalog's files are embedded as they are. +//! not needed: the catalog's files are embedded as they are. The head is +//! embedded as it was converted, f16 or quantized (`convert +//! --quantize-drafter`): a quantized entry's scales and biases sit inside +//! its own payload (`length` spans them, their offsets are relative to the +//! entry's), so the entry moves whole and the runtime reads it from the +//! section as it reads it from the sidecar. //! //! The section is the sidecar's own header, not a Qwen-style one: `arch` is //! the sidecar's (`gemma4-mtp`), `config` is the sidecar's whole config (the @@ -34,7 +41,8 @@ pub struct AddSpeculatorArgs { /// Target `.base` bundle: the model the head drafts for. pub target: PathBuf, /// The head, converted on its own (`basert convert - /// google/gemma-4-E4B-it-assistant` writes a `gemma4-mtp` sidecar). + /// google/gemma-4-E4B-it-assistant` writes a `gemma4-mtp` sidecar), f16 + /// or quantized (`--quantize-drafter --target base-q8`). #[arg(long)] pub head: PathBuf, /// Where to write the embedded bundle (refused if it exists). @@ -139,19 +147,14 @@ fn u64_of(v: &serde_json::Value, key: &str) -> Option { } /// The last blob byte any tensor of the header (main, mmproj, speculator) -/// or any of its sub-regions reaches. +/// reaches. An entry's `length` spans its weights and, when quantized, its +/// scales and biases (sub-region offsets are relative to the entry's), so +/// the entry's own range is the whole of it. fn blob_end(header: &serde_json::Value) -> u64 { let mut end = 0u64; let mut visit = |t: &serde_json::Value| { - for (off, len) in [ - ("offset", "length"), - ("scale_offset", "scale_length"), - ("bias_offset", "bias_length"), - ("awq_scale_offset", "awq_scale_length"), - ] { - if let (Some(o), Some(l)) = (u64_of(t, off), u64_of(t, len)) { - end = end.max(o + l); - } + if let (Some(o), Some(l)) = (u64_of(t, "offset"), u64_of(t, "length")) { + end = end.max(o + l); } }; for t in tensors_of(header) { @@ -174,10 +177,30 @@ fn parse_flags(header: &serde_json::Value) -> Result { } } -fn has_sub_regions(t: &serde_json::Value) -> bool { - ["scale_offset", "bias_offset", "awq_scale_offset"] - .iter() - .any(|k| t.get(k).is_some_and(|v| !v.is_null())) +/// A quantized entry's sub-regions (scales, biases, AWQ scales), each +/// `(offset, length)` relative to the entry's own payload. +const SUB_REGIONS: [(&str, &str, &str); 3] = [ + ("scale_offset", "scale_length", "scales"), + ("bias_offset", "bias_length", "biases"), + ("awq_scale_offset", "awq_scale_length", "AWQ scales"), +]; + +/// Why a sub-region of `t` reaches outside its payload, if one does: the +/// runtime refuses such an entry when it opens the section, so refuse it +/// before the copy. The sum is checked without overflowing. +fn sub_region_overrun(t: &serde_json::Value) -> Option { + let len = u64_of(t, "length").unwrap_or(0); + for (off, sub_len, what) in SUB_REGIONS { + if let (Some(o), Some(l)) = (u64_of(t, off), u64_of(t, sub_len)) { + if l > 0 && (l > len || o > len - l) { + return Some(format!( + "{what} [{o}, {}) outside its payload of {len} bytes", + o.saturating_add(l) + )); + } + } + } + None } /// Embed the head sidecar at `head` in the bundle at `target`, writing the @@ -228,7 +251,7 @@ pub fn add_speculator( ); } - // ── The head: an MTP sidecar of this target's backbone, unquantized ── + // ── The head: an MTP sidecar of this target's backbone, f16 or quantized ── let head_arch = h .header .get("arch") @@ -282,12 +305,16 @@ pub fn add_speculator( if h_tensors.is_empty() { bail!("{}: no tensors", head.display()); } - if let Some(q) = h_tensors.iter().find(|t| has_sub_regions(t)) { - bail!( - "{}: tensor {} is quantized (has sub-regions); a head is embedded unquantized — convert it f16", - head.display(), - q.get("name").and_then(|n| n.as_str()).unwrap_or("?") - ); + // A quantized head's entries move whole (see the module doc); only a + // sub-region that reaches past its payload is refused. + for t in &h_tensors { + if let Some(why) = sub_region_overrun(t) { + bail!( + "{}: tensor {} has its {why}", + head.display(), + t.get("name").and_then(|n| n.as_str()).unwrap_or("?") + ); + } } let h_blob_len = h.file_size - h.blob_start; if blob_end(&h.header) > h_blob_len { @@ -298,13 +325,28 @@ pub fn add_speculator( } // ── Place the head's tensors behind the target's blob ── - let alignment: AlignmentConfig = match t.header.get("alignment") { - Some(a) => serde_json::from_value(a.clone()).context("target alignment")?, - None => AlignmentConfig::default(), + // Each by the stricter of the two files' rules for its region: the + // target's, which placed the target's own tensors, and the head's, + // which placed it as a sidecar (a Metal head's converter puts + // accelerator-region tensors on 16 KiB pages; a catalog target from an + // older converter asks 64 B of them). The first one starts a GPU page + // whatever the rules say: the runtime wraps the section as one buffer + // from its first entry, and Metal needs that base page-aligned — at + // 64 B the whole head falls back to per-tensor copies. + let alignment_of = |o: &Opened, what: &str| -> Result { + Ok(match o.header.get("alignment") { + Some(a) => { + serde_json::from_value(a.clone()).with_context(|| format!("{what} alignment"))? + } + None => AlignmentConfig::default(), + }) }; + let (t_align, h_align) = (alignment_of(&t, "target")?, alignment_of(&h, "head")?); + let align_for = + |region: ComputeRegion| t_align.align_for(region).max(h_align.align_for(region)); let mut cursor = t_blob_len; let mut placed: Vec<(serde_json::Value, u64, u64, u64)> = Vec::with_capacity(h_tensors.len()); // (entry, src offset, dst offset, length) - for src in &h_tensors { + for (i, src) in h_tensors.iter().enumerate() { let region: ComputeRegion = match src.get("compute_region") { Some(r) => serde_json::from_value(r.clone()).context("head tensor compute_region")?, None => ComputeRegion::default(), @@ -313,7 +355,11 @@ pub fn add_speculator( u64_of(src, "offset").context("head tensor offset")?, u64_of(src, "length").context("head tensor length")?, ); - let dst = align_up(cursor, alignment.align_for(region)); + let mut align = align_for(region); + if i == 0 { + align = align.max(align_for(ComputeRegion::Gpu)); + } + let dst = align_up(cursor, align); let mut entry = (*src).clone(); entry["offset"] = serde_json::json!(dst); placed.push((entry, off, dst, len)); @@ -526,7 +572,7 @@ pub(crate) mod fixtures { mod tests { use super::fixtures::*; use super::*; - use base_format::{BaseReader, BaseWriter, TensorDtype, TensorEntry}; + use base_format::{BaseReader, BaseWriter, TensorDtype, TensorEntry, TensorPayload}; /// A tiny Gemma-4-shaped target (two main tensors, one mmproj tensor) /// and a head sidecar over it. @@ -663,6 +709,110 @@ mod tests { ); } + /// A head converted with `--quantize-drafter`: its entries carry scale + /// and bias sub-regions inside their payloads, and they are embedded as + /// they are — the entry equal apart from its new offset, the sub-region + /// offsets still relative to it, the bytes the sidecar's. Its + /// accelerator-region tensors keep the page alignment the head's own + /// converter gave them, though the (older) target asks only 64 B. + #[test] + fn embeds_a_quantized_head_with_its_sub_regions() { + let dir = tempfile::tempdir().unwrap(); + let (target, _) = fixtures(dir.path()); + let head = dir.path().join("q8head.base"); + let mut hdr = header( + "gemma4-mtp", + serde_json::json!({"hidden_size": 4, "backbone_hidden_size": 8, "vocab_size": 16, "num_hidden_layers": 4}), + ); + hdr.alignment.accel_align_log2 = 14; + let mut w = BaseWriter::create(&head, hdr).unwrap(); + // 64 q8 weights, then 8 bytes of scales and 8 of biases, one payload. + let q8 = |name: &str, first: u8| { + let mut p = TensorPayload { + entry: entry(name, vec![16, 4]), + data: (first..first + 80).collect(), + }; + p.entry.dtype = TensorDtype::BaseQ8; + p.entry.compute_region = ComputeRegion::Accelerator; + p.entry.scale_offset = Some(64); + p.entry.scale_length = Some(8); + p.entry.bias_offset = Some(72); + p.entry.bias_length = Some(8); + p.entry.group_size = Some(128); + p + }; + w.add_tensor(q8("output.weight", 0)); + w.add_tensor(payload("output_norm.weight", 8, 0xC2)); + w.add_tensor(q8("layers.0.mlp.down_proj.weight", 100)); + w.finish().unwrap(); + + let out = dir.path().join("out.base"); + let s = add_speculator(&target, &head, &out, Some("q8")).unwrap(); + assert_eq!((s.n_head_tensors, s.head_bytes), (3, 168)); + let hh = BaseReader::open(&head).unwrap(); + let after = BaseReader::open(&out).unwrap(); + let spec = after.header().speculator.as_ref().unwrap(); + assert_eq!(spec.config["head_version"], serde_json::json!("q8")); + for (src, dst) in hh.header().tensors.iter().zip(&spec.tensors) { + let mut moved = src.clone(); + moved.offset = dst.offset; + assert_eq!( + serde_json::to_string(&moved).unwrap(), + serde_json::to_string(dst).unwrap(), + "{}: the entry moves whole", + src.name + ); + assert_eq!( + bytes_of(&head, hh.blob_offset(), src), + bytes_of(&out, after.blob_offset(), dst), + "{}", + dst.name + ); + after.verify_tensor(&dst.name).unwrap(); + } + let q = &spec.tensors[0]; + assert_eq!( + ( + q.dtype, + q.scale_offset, + q.scale_length, + q.bias_offset, + q.bias_length + ), + (TensorDtype::BaseQ8, Some(64), Some(8), Some(72), Some(8)) + ); + let bytes = bytes_of(&out, after.blob_offset(), q); + assert_eq!( + &bytes[64..72], + &(64u8..72).collect::>()[..], + "the scales" + ); + assert_eq!( + &bytes[72..80], + &(72u8..80).collect::>()[..], + "the biases" + ); + // The target's rule (64 B) would have put the second q8 entry right + // behind the norm; the head's (a page) wins, and so does the first + // entry's page start. + assert_eq!(after.header().alignment.accel_align_log2, 6); + assert_eq!( + spec.tensors[0].offset % 16384, + 0, + "the first entry starts a GPU page" + ); + assert_eq!( + spec.tensors[2].offset % 16384, + 0, + "an accelerator tensor keeps the head's page alignment" + ); + assert_eq!( + spec.tensors[1].offset, + spec.tensors[0].offset + 16384, + "the norm follows on the next page (gpu)" + ); + } + #[test] fn refuses_what_it_cannot_embed_into_or_from() { let dir = tempfile::tempdir().unwrap(); @@ -699,8 +849,8 @@ mod tests { .to_string() .contains("but the head drafts for")); - // A quantized head: the sidecar is embedded as it is, and a head - // with scale sub-regions is not the f16 one the rule keeps. + // A quantized head whose scales reach past the entry's payload: the + // runtime would refuse the section, so the embed refuses the head. let q = dir.path().join("q.base"); let mut w = BaseWriter::create( &q, @@ -712,14 +862,14 @@ mod tests { .unwrap(); let mut p = payload("output.weight", 64, 1); p.entry.dtype = TensorDtype::BaseQ4; - p.entry.scale_offset = Some(0); - p.entry.scale_length = Some(0); + p.entry.scale_offset = Some(60); + p.entry.scale_length = Some(8); w.add_tensor(p); w.finish().unwrap(); assert!(add_speculator(&target, &q, &out, None) .unwrap_err() .to_string() - .contains("quantized")); + .contains("scales [60, 68) outside its payload of 64 bytes")); // Embedding twice. add_speculator(&target, &head, &out, None).unwrap(); diff --git a/base-convert/crates/base-convert/src/hub.rs b/base-convert/crates/base-convert/src/hub.rs index 8cbc573..4f004c8 100644 --- a/base-convert/crates/base-convert/src/hub.rs +++ b/base-convert/crates/base-convert/src/hub.rs @@ -79,6 +79,85 @@ fn is_quant_tag(tag: &str) -> bool { ) } +/// The sizes a release names its `.base` files by instead of a quant (the +/// BaseQuant repos' `-XS`/`-Small`/`-Medium`/`-Large`/`-XL`), smallest first. +const SIZE_NAMES: [&str; 5] = ["xs", "small", "medium", "large", "xl"]; + +/// The size a pull takes from such a release when nothing names one. +const DEFAULT_SIZE: &str = "medium"; + +/// The size `name` spells, case aside (`Medium` → `medium`). +fn size_name(name: &str) -> Option<&'static str> { + let n = name.to_ascii_lowercase(); + SIZE_NAMES.iter().copied().find(|s| *s == n) +} + +/// The size a `.base` is named by (`Foo-Medium.base` → `medium`), if any. +fn size_tag(file: &str) -> Option<&'static str> { + size_name(&quant_tag(&base_file_stem(file))) +} + +/// The sizes a repo's `.base` files come in, smallest first, or None when +/// none is named by size. +fn sizes_offered(files: &Grouped) -> Option { + let sizes: Vec<&str> = SIZE_NAMES + .iter() + .copied() + .filter(|s| files.artifacts.iter().any(|f| size_tag(&f.name) == Some(s))) + .collect(); + (!sizes.is_empty()).then(|| sizes.join(", ")) +} + +/// The `.base` a plain pull takes from a repo whose files are named by size +/// and none by quant: the one of [`DEFAULT_SIZE`], else the sole file. +/// None when the repo is not such a release, so the quant rule applies; an +/// incomplete part set of that size is an error saying why, never another +/// size. +fn default_size_file(files: &Grouped) -> Result> { + let quant_named = |n: &str| is_quant_tag(&quant_tag(&base_file_stem(n))); + let sized = files.artifacts.iter().any(|f| size_tag(&f.name).is_some()) + || files.malformed.iter().any(|(n, _)| size_tag(n).is_some()); + if !sized + || files.artifacts.iter().any(|f| quant_named(&f.name)) + || files.malformed.iter().any(|(n, _)| quant_named(n)) + { + return Ok(None); + } + let mediums: Vec<&Artifact> = files + .artifacts + .iter() + .filter(|f| size_tag(&f.name) == Some(DEFAULT_SIZE)) + .collect(); + match mediums.as_slice() { + [one] => return Ok(Some(one.name.clone())), + [] => {} + many => bail!( + "{DEFAULT_SIZE} names several files in this repo ({}); take one with --file", + many.iter() + .map(|f| f.name.as_str()) + .collect::>() + .join(", ") + ), + } + if let Some((name, why)) = files + .malformed + .iter() + .find(|(n, _)| size_tag(n) == Some(DEFAULT_SIZE)) + { + bail!("{name}: the {DEFAULT_SIZE} publication is incomplete ({why})"); + } + if let ([only], []) = (files.artifacts.as_slice(), files.malformed.as_slice()) { + return Ok(Some(only.name.clone())); + } + Ok(None) +} + +/// Whether the pull names a quant (`--profile`, `--target`) rather than +/// taking the default one. +fn quant_named(args: &PullArgs) -> bool { + args.profile.is_some() || args.target.is_some() +} + /// The quant the user is asking for, derived from `--profile` (its filename) /// or, failing that, `--target`. fn quant_token(args: &PullArgs) -> String { @@ -87,7 +166,7 @@ fn quant_token(args: &PullArgs) -> String { return quant_tag(stem); } } - quant_tag(target_str(args.target)) + quant_tag(target_str(args.target_scheme())) } /// `path/to/Foo-Q4.base` → `Foo-Q4`. @@ -144,6 +223,13 @@ fn select_base_file<'a>(files: &'a Grouped, want: &str) -> Result<&'a Artifact> .map(|(n, _)| format!("{} (incomplete)", quant_tag(&base_file_stem(n)))), ) .collect(); + if let Some(sizes) = sizes_offered(files) { + bail!( + "no .base for quant {want:?} in this repo: its files are named by size \ + ({sizes}); take one as `:` or by name with --file: {}", + file_names(files) + ); + } bail!( "no pre-converted .base for quant {want:?} in this repo; it offers: {} \ (or take one by name with --file: {})", @@ -219,6 +305,9 @@ fn base_variant(file: &str, want: &str) -> String { let file_tag = quant_tag(&base_file_stem(file)); if is_quant_tag(&file_tag) { format!("default-{file_tag}") + } else if size_tag(file).is_some() { + // a file named by size is its own variant, as `--file` installs it + named_variant(file) } else { format!("default-{want}") } @@ -336,6 +425,16 @@ fn installed_single_path(reg: &MergedRegistry, id: &str) -> Result = many + .iter() + .copied() + .filter(|r| r.file.as_deref().and_then(size_tag) == Some(DEFAULT_SIZE)) + .collect(); + (sized.len() == 1).then(|| sized[0]) + }); if let Some(one) = pick { return Ok(Some(one.path.clone().with_context(|| { format!("installed model `{id}` has no artifact path") @@ -362,6 +461,9 @@ fn installed_best_variant(reg: &MergedRegistry, id: &str, want: &str) -> Option< if let Some(p) = reg.local.installed_path(id, want) { return Some(p); // exact variant name present } + if let Some(size) = size_name(want) { + return installed_size_path(reg, id, size); + } let want_bits = base_hub::registry::quant_bits(want).unwrap_or(want); let backend = base_hub::registry::CatalogRegistry::client_backend(); let native_prefix = format!("{backend}-"); @@ -397,6 +499,23 @@ fn installed_best_variant(reg: &MergedRegistry, id: &str, want: &str) -> Option< matches[0].path.clone() } +/// The install of `id` that holds the `.base` named by `size` (`:medium`): +/// a file taken by name or as a release's default size installs under its +/// own `file-…` variant, recording the file, which is the only way its size +/// is known. +fn installed_size_path(reg: &MergedRegistry, id: &str, size: &str) -> Option { + reg.local + .list() + .ok()? + .into_iter() + .find(|r| { + r.id == id + && r.variant.starts_with(NAMED_VARIANT_PREFIX) + && r.file.as_deref().and_then(size_tag) == Some(size) + }) + .and_then(|r| r.path) +} + /// Fetch a not-yet-installed model on demand, then return its artifact path. /// Prefers the pre-converted basecompute mirror; otherwise converts the source /// repo on pull. Progress (download + quantization) is shown by `cmd_pull`. @@ -419,7 +538,7 @@ fn auto_pull_and_resolve( cmd_pull(PullArgs { id: want_variant.map_or_else(|| pull_id.clone(), |v| format!("{pull_id}:{v}")), profile: None, - target: TargetScheme::BaseQ4, + target: None, revision: "main".to_string(), file: None, force: false, @@ -526,7 +645,7 @@ fn serve_ready( cmd_pull(PullArgs { id: format!("{id}:{variant}"), profile: None, - target: TargetScheme::BaseQ4, + target: None, revision: "main".to_string(), file: None, force: true, @@ -773,11 +892,11 @@ pub fn cmd_pull(args: PullArgs) -> Result<()> { // Resolved against whichever catalog the attempt is given: the cached one // first, the live one if the cached rows prove to be behind the Hub. with_fresh_catalog_on_mismatch(®, MergedRegistry::load_fresh, |reg| { + let fetcher = HfFetcher::new(cache::hf_staging_dir(®.root))?; if let Some((id, variant)) = args.id.split_once(':') { - let fetcher = HfFetcher::new(cache::hf_staging_dir(®.root))?; return pull_variant(reg, &args, id, variant, &fetcher); } - pull_id(reg, &args) + pull_id(reg, &args, &fetcher) }) } @@ -831,7 +950,7 @@ fn pull_variant( let given = [ args.file.as_ref().map(|_| "--file"), args.profile.as_ref().map(|_| "--profile"), - (args.target != TargetScheme::BaseQ4).then_some("--target"), + args.target.map(|_| "--target"), ]; if let Some(flag) = given.into_iter().flatten().next() { bail!( @@ -845,7 +964,7 @@ fn pull_variant( return Ok(()); } } - let pull = |file: Option, target: TargetScheme| PullArgs { + let pull = |file: Option, target: Option| PullArgs { id: id.to_string(), profile: None, target, @@ -860,7 +979,7 @@ fn pull_variant( let named = pull(Some(file.clone()), args.target); pull_named_file(reg, &named, &file, fetcher) } - VariantSource::Quant(target) => pull_id(reg, &pull(None, target)), + VariantSource::Quant(target) => pull_id(reg, &pull(None, Some(target)), fetcher), } } @@ -953,6 +1072,11 @@ fn variant_source( (None, false) => {} } } + // A size (`medium`): the file of that size in the id's repo, pulled as + // `--file` pulls it. + if let Some(size) = size_name(variant) { + return size_variant_file(reg, id, size, revision, fetcher).map(VariantSource::File); + } if let Some(target) = target_alias(variant) { return Ok(VariantSource::Quant(target)); } @@ -969,10 +1093,70 @@ fn variant_source( }; bail!( "{id} has no variant {variant:?}{listed}. Name one `basert list --remote` shows, \ - a quant (q4, q8, bf16, …), or a file with --file" + a quant (q4, q8, bf16, …), a size (xs, small, medium, large, xl), or a file \ + with --file" ) } +/// Where the files of `id` are: the repos (with revisions) its catalog rows +/// use, else the id's own repo at `revision`. +fn repo_sources(reg: &MergedRegistry, id: &str, revision: &str) -> Vec<(String, String)> { + let mut sources: Vec<(String, String)> = Vec::new(); + for e in reg.catalog.rows(id) { + let at = (e.hf_repo.clone(), e.revision.clone()); + if !sources.contains(&at) { + sources.push(at); + } + } + if sources.is_empty() { + sources.push((id.to_string(), revision.to_string())); + } + sources +} + +/// The `.base` of `id` named by `size` (`:medium` → `Foo-Medium.base`), +/// looked for where `--file` looks. A size the release does not come in +/// names the sizes it does; one found in several places must be given as a +/// file with its path. +fn size_variant_file( + reg: &MergedRegistry, + id: &str, + size: &str, + revision: &str, + fetcher: &dyn Fetcher, +) -> Result { + let sources = repo_sources(reg, id, revision); + let mut names = Vec::new(); + let mut sizes = Vec::new(); + for (repo, revision) in &sources { + let files = list_base_files(fetcher, repo, revision)?; + if let Some(offered) = sizes_offered(&files) { + sizes.push(offered); + } + names.extend(files.artifacts.iter().map(|f| f.name.clone())); + } + let hits: Vec<&String> = names.iter().filter(|n| size_tag(n) == Some(size)).collect(); + match hits.as_slice() { + [one] => Ok((*one).clone()), + [] if sizes.is_empty() => bail!( + "{id} has no {size} .base: its files are not named by size ({}); name a \ + variant `basert list --remote` shows, a quant, or a file with --file", + names.join(", ") + ), + [] => bail!( + "{id} has no {size} .base; it comes in: {}", + sizes.join(", ") + ), + many => bail!( + "{size} names several files of {id} ({}); give one with --file and its path", + many.iter() + .map(|n| n.as_str()) + .collect::>() + .join(", ") + ), + } +} + /// The repo file the `file-…` variant `variant` was named for (see /// [`named_variant`]), looked for where `--file` looks: the repos the id's /// catalog rows use, else the id's own. A file the catalog publishes installs @@ -984,16 +1168,7 @@ fn named_variant_file( revision: &str, fetcher: &dyn Fetcher, ) -> Result { - let mut sources: Vec<(String, String)> = Vec::new(); - for e in reg.catalog.rows(id) { - let at = (e.hf_repo.clone(), e.revision.clone()); - if !sources.contains(&at) { - sources.push(at); - } - } - if sources.is_empty() { - sources.push((id.to_string(), revision.to_string())); - } + let sources = repo_sources(reg, id, revision); let mut names = Vec::new(); for (repo, revision) in &sources { let files = list_base_files(fetcher, repo, revision)?; @@ -1023,16 +1198,17 @@ fn named_variant_file( } /// `basert pull `, with no `:variant`. -fn pull_id(reg: &MergedRegistry, args: &PullArgs) -> Result<()> { +fn pull_id(reg: &MergedRegistry, args: &PullArgs, fetcher: &dyn Fetcher) -> Result<()> { let root = reg.root.clone(); let want = quant_token(args); if let Some(file) = &args.file { - let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; - return pull_named_file(reg, args, file, &fetcher); + return pull_named_file(reg, args, file, fetcher); } let r = reg.resolve(&args.id, &args.revision, Some(&want), args.force)?; - if args.dry_run { + // A raw repo's plan is read off its listing (which file, or that none + // fits), as the pull would; the rest is settled here. + if args.dry_run && !matches!(r, ModelRef::HuggingFace { .. }) { print_plan(&r, &want); return Ok(()); } @@ -1065,18 +1241,16 @@ fn pull_id(reg: &MergedRegistry, args: &PullArgs) -> Result<()> { // installed_best_variant / the catalog resolver's bit-aware match. let want_bits = base_hub::registry::quant_bits(&want).unwrap_or(&want); if base_hub::registry::quant_bits(variant).unwrap_or(variant) == want_bits { - let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; - pull_catalog(&root, &r, &fetcher) + pull_catalog(&root, &r, fetcher) } else { - let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; - let base_files = list_base_files(&fetcher, hf_repo, revision)?; + let base_files = list_base_files(fetcher, hf_repo, revision)?; pull_base_direct( &root, args, id, hf_repo, revision, - &fetcher, + fetcher, &base_files, None, None, @@ -1087,28 +1261,78 @@ fn pull_id(reg: &MergedRegistry, args: &PullArgs) -> Result<()> { // (e.g. the basecompute org), download the matching one directly — no // local conversion. Otherwise treat it as source safetensors. ModelRef::HuggingFace { id, repo, revision } => { - let fetcher = HfFetcher::new(cache::hf_staging_dir(&root))?; - let base_files = list_base_files(&fetcher, repo, revision)?; - // A repo holding only a half-uploaded split bundle is still a - // `.base` repo: it gets the incomplete-publication error, not a - // conversion attempt. - if base_files.artifacts.is_empty() && base_files.malformed.is_empty() { - pull_and_convert(&root, args, id, repo, revision) - } else { - pull_base_direct( - &root, - args, - id, - repo, - revision, - &fetcher, - &base_files, - None, - None, - ) - } + pull_hf_repo(reg, args, id, repo, revision, fetcher) + } + } +} + +/// `basert pull ` of a raw Hub repo: its pre-converted `.base` when it +/// hosts any, else the source safetensors converted here. +fn pull_hf_repo( + reg: &MergedRegistry, + args: &PullArgs, + id: &str, + repo: &str, + revision: &str, + fetcher: &dyn Fetcher, +) -> Result<()> { + let base_files = list_base_files(fetcher, repo, revision)?; + // A repo holding only a half-uploaded split bundle is still a + // `.base` repo: it gets the incomplete-publication error, not a + // conversion attempt. + if base_files.artifacts.is_empty() && base_files.malformed.is_empty() { + if args.dry_run { + let r = ModelRef::HuggingFace { + id: id.to_string(), + repo: repo.to_string(), + revision: revision.to_string(), + }; + print_plan(&r, &quant_token(args)); + return Ok(()); } + return pull_and_convert(®.root, args, id, repo, revision); } + // A release in sizes (the BaseQuant repos): with no file and no quant + // named, its default size, pulled as `--file` pulls it, so both land in + // one install. + if args.file.is_none() && !quant_named(args) { + if let Some(file) = default_size_file(&base_files)? { + eprintln!( + "{id}: a release in sizes; taking {file} (`{id}:` or --file takes another)" + ); + let named = PullArgs { + id: args.id.clone(), + profile: None, + target: args.target, + revision: args.revision.clone(), + file: Some(file.clone()), + force: args.force, + dry_run: args.dry_run, + }; + return pull_named_file(reg, &named, &file, fetcher); + } + } + if args.dry_run { + let want = quant_token(args); + let artifact = select_base_file(&base_files, &want)?; + println!( + "plan: pull {} from {repo}@{revision} → {id} [{}] (no conversion)", + artifact.name, + base_variant(&artifact.name, &want) + ); + return Ok(()); + } + pull_base_direct( + ®.root, + args, + id, + repo, + revision, + fetcher, + &base_files, + None, + None, + ) } /// `pull --file`: download the named `.base` from the id's repo (the catalog's @@ -1630,7 +1854,7 @@ fn pull_and_convert( output: Some(out.clone()), mtp_head: None, mtp_head_version: None, - target: args.target, + target: args.target_scheme(), calib: None, calib_tokens: 512, awq_mode: AwqMode::Full, @@ -1725,7 +1949,7 @@ fn choose_profile( .to_string(); return Ok((variant, Some(p.clone()), None)); } - if let Some((name, json)) = bundled_profile_for_target(args.target) { + if let Some((name, json)) = bundled_profile_for_target(args.target_scheme()) { let mut tf = tempfile::Builder::new() .prefix("base-profile-") .suffix(".json") @@ -1735,7 +1959,7 @@ fn choose_profile( return Ok((name.to_string(), Some(path), Some(tf))); } // No generic profile for this scheme → flat --target, variant = scheme. - Ok((target_str(args.target).to_string(), None, None)) + Ok((target_str(args.target_scheme()).to_string(), None, None)) } /// The drafter sidecar architecture the snapshot's `config.json` names @@ -2344,7 +2568,7 @@ mod tests { let args = PullArgs { id: "basecompute/m".into(), profile: None, - target: TargetScheme::BaseQ4, + target: None, revision: "main".into(), file: None, force: false, @@ -2414,7 +2638,7 @@ mod tests { let args = PullArgs { id: "basecompute/m".into(), profile: None, - target: TargetScheme::BaseQ8, + target: Some(TargetScheme::BaseQ8), revision: "main".into(), file: None, force: false, @@ -2564,7 +2788,7 @@ mod tests { PullArgs { id: id.into(), profile: None, - target: TargetScheme::BaseQ4, + target: None, revision: "main".into(), file: Some(file.into()), force: false, @@ -2648,7 +2872,10 @@ mod tests { let base_files = list_base_files(&fetcher, "basecompute/m", "main").unwrap(); let err = select_base_file(&base_files, "q4").unwrap_err().to_string(); assert!( - err.contains("--file") && err.contains("m-Medium.base"), + err.contains("named by size (small, medium)") + && err.contains("`:`") + && err.contains("--file") + && err.contains("m-Medium.base"), "{err}" ); @@ -2680,6 +2907,271 @@ mod tests { assert!(installed_best_variant(®, "basecompute/m", "default-q4").is_none()); } + fn plain(id: &str) -> PullArgs { + with_variant(id, false) + } + + fn pull_repo(reg: &MergedRegistry, fetcher: &dyn Fetcher, args: &PullArgs) -> Result<()> { + pull_hf_repo(reg, args, &args.id, &args.id, "main", fetcher) + } + + /// A release in sizes (the BaseQuant repos): a plain pull takes its + /// medium, under the variant `--file` would install it in, so the two + /// share one install; a quant named instead is refused naming the sizes. + #[test] + fn a_release_in_sizes_pulls_its_medium_by_default() { + let tmp = tempfile::tempdir().unwrap(); + let id = "basecompute/m"; + fixture_repo( + tmp.path(), + id, + &[ + ("m-XS.base", b"xs"), + ("m-Small.base", b"small"), + ("m-Medium.base", b"medium"), + ("m-Large.base", b"large"), + ("m-XL.base", b"xl"), + ("m-Medium.gguf", b"gguf"), + ], + ); + let fetcher = MockFetcher::new(tmp.path()); + let reg = MergedRegistry::new(tmp.path().join("cache"), catalog("")); + let medium = named_variant("m-Medium.base"); + + pull_repo(®, &fetcher, &plain(id)).unwrap(); + let path = reg.local.installed_path(id, &medium).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), b"medium"); + assert_eq!(held(®, id, &medium).as_deref(), Some("m-Medium.base")); + assert_eq!(reg.local.list().unwrap().len(), 1); + + // installed: a second plain pull, a `--file` pull and a `:medium` + // pull all find it without a download + let listing = Listing(vec!["m-Small.base", "m-Medium.base", "m-Large.base"]); + pull_repo(®, &listing, &plain(id)).unwrap(); + pull_repo(®, &listing, &with_variant(id, true)).unwrap(); + pull_named(®, &listing, &named("m-Medium.base", false, id)).unwrap(); + pull_with_variant(®, &listing, &with_variant("basecompute/m:medium", false)).unwrap(); + pull_with_variant(®, &listing, &with_variant("basecompute/m:Medium", false)).unwrap(); + assert_eq!(reg.local.list().unwrap().len(), 1); + + // and `serve ` / `serve :medium` resolve to it + assert_eq!( + installed_single_path(®, id).unwrap().as_deref(), + Some(path.as_path()) + ); + assert_eq!( + installed_best_variant(®, id, "medium").as_deref(), + Some(path.as_path()) + ); + assert_eq!( + installed_best_variant(®, id, "Medium").as_deref(), + Some(path.as_path()) + ); + assert!(installed_best_variant(®, id, "small").is_none()); + assert!(installed_best_variant(®, id, "default-q4").is_none()); + + // a size names its file; one the release lacks names the sizes it has + assert_eq!( + variant_source(®, id, "small", "main", &fetcher).unwrap(), + VariantSource::File("m-Small.base".into()) + ); + pull_with_variant(®, &fetcher, &with_variant("basecompute/m:small", false)).unwrap(); + let small = named_variant("m-Small.base"); + assert_eq!( + std::fs::read(reg.local.installed_path(id, &small).unwrap()).unwrap(), + b"small" + ); + assert_eq!( + installed_best_variant(®, id, "small").unwrap(), + reg.local.installed_path(id, &small).unwrap() + ); + // with several sizes installed, the bare id is still the medium + assert_eq!( + installed_single_path(®, id).unwrap().as_deref(), + Some(path.as_path()) + ); + + // a quant named instead of a size is refused, naming the sizes + let mut q8 = plain(id); + q8.target = Some(TargetScheme::BaseQ8); + let err = pull_repo(®, &fetcher, &q8).unwrap_err().to_string(); + assert!( + err.contains("named by size (xs, small, medium, large, xl)") + && err.contains("`:`"), + "{err}" + ); + q8.dry_run = true; + let err = pull_repo(®, &listing, &q8).unwrap_err().to_string(); + assert!(err.contains("named by size"), "{err}"); + } + + /// A dry run of a raw repo reads its plan off the listing: the file a + /// pull would take and the variant it would land in, nothing fetched. + #[test] + fn a_dry_run_of_a_raw_repo_names_its_file() { + let tmp = tempfile::tempdir().unwrap(); + let reg = MergedRegistry::new(tmp.path().join("cache"), catalog("")); + let listing = Listing(vec!["m-Q4.base", "m-Q8.base"]); + let mut args = plain("org/m"); + args.dry_run = true; + pull_repo(®, &listing, &args).unwrap(); + args.target = Some(TargetScheme::BaseQ8); + pull_repo(®, &listing, &args).unwrap(); + args.target = Some(TargetScheme::BaseQ6); + let err = pull_repo(®, &listing, &args).unwrap_err().to_string(); + assert!(err.contains("it offers: q4, q8"), "{err}"); + // a source repo's plan is the conversion it would run, nothing fetched + let sources = Listing(vec!["config.json", "model.safetensors"]); + pull_repo(®, &sources, &args).unwrap(); + assert!(reg.local.list().unwrap().is_empty()); + } + + /// The sizes rule holds only for a release named by size and nothing + /// else: one file is taken whatever its size, no medium among several + /// is an error, a quant-named file beside a size keeps the quant rule, + /// and a half-uploaded medium is reported, never another size. + #[test] + fn a_release_in_sizes_without_a_medium_or_beside_quants() { + let tmp = tempfile::tempdir().unwrap(); + let fetcher = MockFetcher::new(tmp.path()); + let reg = MergedRegistry::new(tmp.path().join("cache"), catalog("")); + + fixture_repo(tmp.path(), "org/one", &[("one-Small.base", b"small")]); + pull_repo(®, &fetcher, &plain("org/one")).unwrap(); + let small = named_variant("one-Small.base"); + assert_eq!( + std::fs::read(reg.local.installed_path("org/one", &small).unwrap()).unwrap(), + b"small" + ); + assert!(installed_best_variant(®, "org/one", "small").is_some()); + + fixture_repo( + tmp.path(), + "org/two", + &[("two-Small.base", b"small"), ("two-Large.base", b"large")], + ); + let err = pull_repo(®, &fetcher, &plain("org/two")) + .unwrap_err() + .to_string(); + assert!(err.contains("named by size (small, large)"), "{err}"); + let err = variant_source(®, "org/two", "medium", "main", &fetcher) + .unwrap_err() + .to_string(); + assert!(err.contains("it comes in: small, large"), "{err}"); + + fixture_repo( + tmp.path(), + "org/mixed", + &[("mixed-Q4.base", b"q4"), ("mixed-Medium.base", b"medium")], + ); + pull_repo(®, &fetcher, &plain("org/mixed")).unwrap(); + assert_eq!( + std::fs::read(reg.local.installed_path("org/mixed", "default-q4").unwrap()).unwrap(), + b"q4" + ); + // its medium is still reachable by size + assert_eq!( + variant_source(®, "org/mixed", "medium", "main", &fetcher).unwrap(), + VariantSource::File("mixed-Medium.base".into()) + ); + + let files = Grouped { + artifacts: vec![Artifact::whole("m-Small.base")], + malformed: vec![("m-Medium.base".into(), "index 1 is missing".into())], + }; + let err = default_size_file(&files).unwrap_err().to_string(); + assert!( + err.contains("the medium publication is incomplete"), + "{err}" + ); + let files = Grouped { + artifacts: vec![Artifact::whole("m-Q4.base"), Artifact::whole("m-Q8.base")], + malformed: vec![], + }; + assert_eq!(default_size_file(&files).unwrap(), None); + let files = Grouped { + artifacts: vec![Artifact::whole("Foo.base")], + malformed: vec![], + }; + assert_eq!(default_size_file(&files).unwrap(), None); + } + + /// A quant named is a quant, the default one included: `--target base-q4` + /// and `:q4` ask for a q4, which a release in sizes has none of, and are + /// refused as any quant is; only a pull naming no quant takes the Medium. + #[test] + fn an_explicit_q4_is_a_quant_asked_for_not_the_default_size() { + let tmp = tempfile::tempdir().unwrap(); + let id = "basecompute/m"; + fixture_repo( + tmp.path(), + id, + &[ + ("m-Small.base", b"small"), + ("m-Medium.base", b"medium"), + ("m-Large.base", b"large"), + ], + ); + let fetcher = MockFetcher::new(tmp.path()); + let reg = MergedRegistry::new(tmp.path().join("cache"), catalog("")); + // as the command line gives them: no --target is no quant named + let parse = |argv: &[&str]| ::try_parse_from(argv).unwrap(); + assert_eq!(parse(&["pull", id]).target, None); + let q4 = parse(&["pull", id, "--target", "base-q4"]); + assert_eq!(q4.target, Some(TargetScheme::BaseQ4)); + let err = pull_repo(®, &fetcher, &q4).unwrap_err().to_string(); + assert!( + err.contains("named by size (small, medium, large)"), + "{err}" + ); + let err = pull_with_variant(®, &fetcher, &with_variant("basecompute/m:q4", false)) + .unwrap_err() + .to_string(); + assert!(err.contains("named by size"), "{err}"); + assert!(reg.local.list().unwrap().is_empty()); + pull_repo(®, &fetcher, &parse(&["pull", id])).unwrap(); + assert_eq!(reg.local.list().unwrap().len(), 1); + } + + /// Two models of one size in a repo are two: the default pick refuses to + /// choose between them, as `:medium` does, and asks for a file. + #[test] + fn two_mediums_are_refused_by_the_default_as_by_the_size() { + let tmp = tempfile::tempdir().unwrap(); + let id = "org/two"; + fixture_repo( + tmp.path(), + id, + &[ + ("model-A-Medium.base", b"a"), + ("model-B-Medium.base", b"b"), + ("model-A-Small.base", b"s"), + ], + ); + let fetcher = MockFetcher::new(tmp.path()); + let reg = MergedRegistry::new(tmp.path().join("cache"), catalog("")); + let err = pull_repo(®, &fetcher, &plain(id)) + .unwrap_err() + .to_string(); + assert!( + err.contains("medium names several files") + && err.contains("model-A-Medium.base") + && err.contains("model-B-Medium.base") + && err.contains("--file"), + "{err}" + ); + let err = variant_source(®, id, "medium", "main", &fetcher) + .unwrap_err() + .to_string(); + assert!(err.contains("medium names several files"), "{err}"); + assert!(reg.local.list().unwrap().is_empty()); + // the one Small is still its size's + assert_eq!( + variant_source(®, id, "small", "main", &fetcher).unwrap(), + VariantSource::File("model-A-Small.base".into()) + ); + } + /// A cached file answers only for itself: the name is resolved against /// the repo first, and whatever else is in the variant is refused. #[test] @@ -3191,7 +3683,7 @@ mod tests { PullArgs { id: id.into(), profile: None, - target: TargetScheme::BaseQ4, + target: None, revision: "main".into(), file: None, force: false, @@ -3354,10 +3846,27 @@ mod tests { VariantSource::Quant(t) ); } + // a size is the file of that size in the id's repo; one the release + // lacks names the sizes it has + assert_eq!( + variant_source(®, "basecompute/cat", "medium", "main", &fetcher).unwrap(), + VariantSource::File("cat-Medium.base".into()) + ); + assert_eq!( + variant_source(®, "basecompute/cat", "Medium", "main", &fetcher).unwrap(), + VariantSource::File("cat-Medium.base".into()) + ); + let err = variant_source(®, "basecompute/cat", "large", "main", &fetcher) + .unwrap_err() + .to_string(); + assert!( + err.contains("has no large .base; it comes in: medium"), + "{err}" + ); + // anything else names the variants this client can pull, a name // that merely contains or ends in a width included for v in [ - "large", "default-q8-custom", "not-a-variant-f16", "typo-q8", @@ -3379,7 +3888,7 @@ mod tests { let mut file = with_variant("basecompute/cat:default-q8", true); file.file = Some("cat-Q8.base".into()); let mut target = with_variant("basecompute/cat:default-q8", true); - target.target = TargetScheme::BaseQ8; + target.target = Some(TargetScheme::BaseQ8); let mut profile = with_variant("basecompute/cat:default-q8", true); profile.profile = Some("default-q8.json".into()); for (args, flag) in [ @@ -3511,7 +4020,7 @@ mod tests { let args = PullArgs { id: "basecompute/m".into(), profile: None, - target: TargetScheme::BaseQ4, + target: None, revision: "main".into(), file: None, force: false, @@ -3880,7 +4389,7 @@ mod tests { let mk = |t| PullArgs { id: "x/y".into(), profile: None, - target: t, + target: Some(t), revision: "main".into(), file: None, force: false, diff --git a/base-convert/crates/base-convert/src/main.rs b/base-convert/crates/base-convert/src/main.rs index d48bfbb..9644e8b 100644 --- a/base-convert/crates/base-convert/src/main.rs +++ b/base-convert/crates/base-convert/src/main.rs @@ -406,22 +406,26 @@ struct PullArgs { /// Model id: `basecompute/` (pre-converted, from the catalog) or /// `org/model` (a raw HF repo, downloaded + converted locally). Add /// `:` for that variant, as `basert list` names it - /// (`default-q8`, `cuda-q4mix`, `file-…`), or a quant (`q8`). + /// (`default-q8`, `cuda-q4mix`, `file-…`), a quant (`q8`), or a size + /// (`small`) of a release that comes in sizes (`-XS`, `-Small`, + /// `-Medium`, `-Large`, `-XL`); such a release's medium is the default. id: String, /// Override the auto-selected quant profile (path to a profile JSON). #[arg(long)] profile: Option, - /// Target quant scheme for convert-on-pull when no profile applies. - #[arg(long, value_enum, default_value_t = TargetScheme::BaseQ4)] - target: TargetScheme, + /// Target quant scheme for convert-on-pull when no profile applies + /// (base-q4 when not given). Given, it names the quant a pre-converted + /// repo's file is picked by, so a release in sizes refuses it. + #[arg(long, value_enum)] + target: Option, /// HF revision / branch / tag. #[arg(long, default_value = "main")] revision: String, /// Take this `.base` file from the repo (e.g. `Foo-Medium.base`) rather - /// than picking one by quant — for repos whose `.base` files are not - /// named by quant. A file the catalog lists installs under its catalog - /// variant; any other under one of its own (`file-foo-medium-`), - /// which `basert list --json` shows beside the file. + /// than picking one by quant or size. A file the catalog lists installs + /// under its catalog variant; any other under one of its own + /// (`file-foo-medium-`), which `basert list --json` shows beside + /// the file. #[arg(long)] file: Option, /// Re-download / re-convert even if already cached. @@ -432,6 +436,13 @@ struct PullArgs { dry_run: bool, } +impl PullArgs { + /// The quant scheme the pull converts to: the one named, else base-q4. + fn target_scheme(&self) -> TargetScheme { + self.target.unwrap_or(TargetScheme::BaseQ4) + } +} + #[derive(Parser, Debug)] struct CatalogScanArgs { /// Organization to scan. diff --git a/bindings/node/package.json b/bindings/node/package.json index 538c54c..14a6634 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@baseRT/node", - "version": "0.4.4", + "version": "0.4.5", "private": true, "description": "Node.js bindings for BaseRT — LLM inference engine for Apple Silicon (Metal)", "main": "dist/index.js", diff --git a/bindings/python/baseRT/__init__.py b/bindings/python/baseRT/__init__.py index 5b702d0..2d7a27a 100644 --- a/bindings/python/baseRT/__init__.py +++ b/bindings/python/baseRT/__init__.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union -__version__ = "0.4.4" +__version__ = "0.4.5" # --------------------------------------------------------------------------- # Library loading diff --git a/bindings/python/setup.py b/bindings/python/setup.py index b393769..604a7f3 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -6,7 +6,7 @@ setup( name="baseRT", - version="0.4.4", + version="0.4.5", description="Python bindings for the BaseRT LLM inference engine (Apple Silicon / Metal)", long_description=long_description, long_description_content_type="text/markdown", diff --git a/bindings/rust/baseRT-sys/Cargo.toml b/bindings/rust/baseRT-sys/Cargo.toml index b9f81be..f9763d3 100644 --- a/bindings/rust/baseRT-sys/Cargo.toml +++ b/bindings/rust/baseRT-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT-sys" -version = "0.4.4" +version = "0.4.5" edition = "2021" description = "Raw FFI bindings for the BaseRT LLM inference engine" license = "Apache-2.0" diff --git a/bindings/rust/baseRT/Cargo.toml b/bindings/rust/baseRT/Cargo.toml index c01cf36..6389ef7 100644 --- a/bindings/rust/baseRT/Cargo.toml +++ b/bindings/rust/baseRT/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT" -version = "0.4.4" +version = "0.4.5" edition = "2021" description = "Safe Rust bindings for the BaseRT LLM inference engine (Apple Silicon)" license = "Apache-2.0" diff --git a/bindings/swift/Sources/CBaseRT/include/baseRT.h b/bindings/swift/Sources/CBaseRT/include/baseRT.h index 4aa2b70..1032e4f 100644 --- a/bindings/swift/Sources/CBaseRT/include/baseRT.h +++ b/bindings/swift/Sources/CBaseRT/include/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 4 +#define BASERT_VERSION_PATCH 5 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. diff --git a/include/baseRT/baseRT.h b/include/baseRT/baseRT.h index 4aa2b70..1032e4f 100644 --- a/include/baseRT/baseRT.h +++ b/include/baseRT/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 4 +#define BASERT_VERSION_PATCH 5 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks.