diff --git a/Cargo.lock b/Cargo.lock index b506de2..d2328ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -421,7 +421,7 @@ dependencies = [ "byteorder", "candle-core", "candle-nn", - "fancy-regex 0.18.0", + "fancy-regex", "num-traits", "rand 0.9.5", "rayon", @@ -835,9 +835,9 @@ dependencies = [ [[package]] name = "daachorse" -version = "1.0.1" +version = "3.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f55d7153ba3b507595872a3874803f07a8a81d1e888abed8e5db7da0597d6e2" +checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d" [[package]] name = "darling" @@ -1115,17 +1115,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" -[[package]] -name = "fancy-regex" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - [[package]] name = "fancy-regex" version = "0.18.0" @@ -1166,7 +1155,7 @@ dependencies = [ "rusty_alloc-api", "serde", "serde_json", - "tokenizers 0.23.1", + "tokenizers 0.23.2", ] [[package]] @@ -1205,7 +1194,7 @@ dependencies = [ "rusty_alloc-api", "serde", "serde_json", - "tokenizers 0.23.1", + "tokenizers 0.23.2", ] [[package]] @@ -1318,7 +1307,7 @@ dependencies = [ "rayon", "rustfft", "serde_json", - "tokenizers 0.23.1", + "tokenizers 0.23.2", ] [[package]] @@ -2993,28 +2982,6 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" -[[package]] -name = "onig" -version = "6.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" -dependencies = [ - "bitflags 2.13.1", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "openssl-probe" version = "0.2.1" @@ -3845,6 +3812,12 @@ dependencies = [ "rusty_alloc", ] +[[package]] +name = "rusty_expressions" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a959a92cbe571ca9cded420431036ba3db3ba99d2387aecd0879b36f42477015" + [[package]] name = "rusty_h264" version = "0.8.0" @@ -4505,8 +4478,7 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokenizers" version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +source = "git+https://github.com/Remade-With-Rust/tokenizers.git?branch=v0.22-rusty-expressions#cae193a2f70c56226786796ae6dbe85d2c2b5217" dependencies = [ "ahash", "aho-corasick", @@ -4519,13 +4491,13 @@ dependencies = [ "log", "macro_rules_attribute", "monostate", - "onig", "paste", "rand 0.9.5", "rayon", "rayon-cond", "regex", "regex-syntax", + "rusty_expressions", "serde", "serde_json", "spm_precompiled", @@ -4537,9 +4509,8 @@ dependencies = [ [[package]] name = "tokenizers" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44e5bea67576e04b6ff8564c5d9e09c2ef0cf476502245f2f120e497769d3112" +version = "0.23.2" +source = "git+https://github.com/Remade-With-Rust/tokenizers.git?branch=rusty-expressions-deploy#ef352c7ce3dd00b546a09aa06ab9cdf882cd045d" dependencies = [ "ahash", "compact_str", @@ -4547,19 +4518,18 @@ dependencies = [ "dary_heap", "derive_builder", "esaxx-rs", - "fancy-regex 0.17.0", "getrandom 0.3.4", "itertools", "log", "macro_rules_attribute", "monostate", - "onig", "paste", "rand 0.9.5", "rayon", "rayon-cond", "regex", "regex-syntax", + "rusty_expressions", "serde", "serde_json", "spm_precompiled", diff --git a/Cargo.toml b/Cargo.toml index 4cd2a8a..be62e22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,7 +35,14 @@ candle-nn = "0.11" candle-transformers = "0.11" # ---- model loading ---- -tokenizers = { version = "0.23", default-features = false, features = ["onig"] } +# `rusty-expressions` instead of `onig`. `onig` reaches libonig through +# `onig_sys`, i.e. C compiled by `cc`: a C toolchain in every build of this +# workspace and, because Cargo features are additive across the graph, in every +# downstream consumer too -- including ones that had explicitly opted out. +# `rusty_expressions` is Oniguruma remade in pure Rust with the same semantics +# (pre-tokenizer output verified byte-identical against libonig). +# Pending upstream: huggingface/tokenizers#2368. +tokenizers = { version = "0.23", default-features = false, features = ["rusty-expressions"] } rustfft = "6" hf-hub = { version = "1", default-features = false, features = ["blocking", "rustls-tls"] } @@ -98,3 +105,14 @@ lto = "thin" # cache, downloads without hash verification - this decision expires WITH them # and overflow-checks goes back on. codegen-units = 1 + +# Two independent sources of libonig had to be replaced, not one: +# * this workspace's own `tokenizers 0.23`, and +# * `candle-core 0.11`, which pins `tokenizers 0.22` with `features = +# ["onig"]` as a non-optional dependency on every non-wasm target. +# A single [patch] entry cannot cover two semver-incompatible versions, so both +# are patched. Drop the 0.23 entry once huggingface/tokenizers#2368 lands, and +# the 0.22 entry once candle stops hardcoding the onig backend. +[patch.crates-io] +tokenizers = { git = "https://github.com/Remade-With-Rust/tokenizers.git", branch = "rusty-expressions-deploy" } +tokenizers_022 = { package = "tokenizers", git = "https://github.com/Remade-With-Rust/tokenizers.git", branch = "v0.22-rusty-expressions" } diff --git a/crates/ffai-carmenta/Cargo.toml b/crates/ffai-carmenta/Cargo.toml index 0d67690..b446afb 100644 --- a/crates/ffai-carmenta/Cargo.toml +++ b/crates/ffai-carmenta/Cargo.toml @@ -58,10 +58,10 @@ tokenizers = { workspace = true } # `--cfg getrandom_backend` that .cargo/config.toml supplies; it errors if # either half is missing. Arrives here through tokenizers -> ahash. getrandom = { version = "0.3", features = ["wasm_js"] } -# `onig` is Oniguruma, a C regex engine: there is no `cc` for wasm32, and a C -# dependency is not something this house ships anyway. `unstable_wasm` swaps it -# for fancy-regex, which is pure Rust. -tokenizers = { version = "0.23", default-features = false, features = ["unstable_wasm"] } +# The workspace uses `rusty-expressions` everywhere now -- Oniguruma remade in +# pure Rust -- so wasm32 no longer needs a DIFFERENT engine from native, only +# the getrandom plumbing that the `wasm` feature carries. +tokenizers = { version = "0.23", default-features = false, features = ["rusty-expressions", "wasm"] } [dev-dependencies] # Examples only, never the library. `ffai-cli` sets mimalloc as its global