diff --git a/Cargo.toml b/Cargo.toml index 2728b63..fdbe55b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "model2vec-rs" version = "0.2.1" -edition = "2021" +edition = "2024" description = "Official Rust Implementation of Model2Vec" readme = "README.md" license-file = "LICENSE" diff --git a/src/model.rs b/src/model.rs index 990de8b..9cc2a4b 100644 --- a/src/model.rs +++ b/src/model.rs @@ -1,13 +1,11 @@ -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result, anyhow}; use half::f16; #[cfg(all(feature = "hf-hub", not(feature = "local-only")))] -use hf_hub::api::sync::{Api, ApiRepo}; +use hf_hub::api::sync::{Api, ApiBuilder, ApiRepo}; use ndarray::{Array2, ArrayView2, CowArray, Ix2}; -use safetensors::{tensor::Dtype, SafeTensors}; +use safetensors::{SafeTensors, tensor::Dtype}; use serde_json::Value; use std::borrow::Cow; -#[cfg(all(feature = "hf-hub", not(feature = "local-only")))] -use std::env; use std::{ fs, path::{Path, PathBuf}, @@ -472,28 +470,14 @@ fn resolve_model_files>( #[cfg(all(feature = "hf-hub", not(feature = "local-only")))] fn download_model_files(repo_id: &str, token: Option<&str>, subfolder: Option<&str>) -> Result { - let previous = token.and_then(|_| env::var_os("HF_HUB_TOKEN")); - if let Some(tok) = token { - env::set_var("HF_HUB_TOKEN", tok); - } - - let result = (|| { - let api = Api::new().context("hf-hub API init failed")?; - let repo = api.model(repo_id.to_owned()); - let prefix = subfolder.map(|s| format!("{s}/")).unwrap_or_default(); - resolve_hub_model_files(&repo, &prefix) - .with_context(|| format!("could not load '{repo_id}' from HuggingFace Hub")) - })(); - - if token.is_some() { - if let Some(value) = previous { - env::set_var("HF_HUB_TOKEN", value); - } else { - env::remove_var("HF_HUB_TOKEN"); - } + let api = match token { + Some(token) => ApiBuilder::new().with_token(Some(token.to_owned())).build(), + None => Api::new(), } - - result + .context("hf-hub API init failed")?; + let repo = api.model(repo_id.to_owned()); + let prefix = subfolder.map(|s| format!("{s}/")).unwrap_or_default(); + resolve_hub_model_files(&repo, &prefix).with_context(|| format!("could not load '{repo_id}' from HuggingFace Hub")) } #[cfg(test)]