diff --git a/Sources/SwiftLM/Server.swift b/Sources/SwiftLM/Server.swift index 526caa8..9ec27fe 100644 --- a/Sources/SwiftLM/Server.swift +++ b/Sources/SwiftLM/Server.swift @@ -49,9 +49,86 @@ private struct TransformersTokenizerLoader: TokenizerLoader, Sendable { init(modelId: String = "this model") { self.modelId = modelId } func load(from directory: URL) async throws -> any MLXLMCommon.Tokenizer { - let t = try await AutoTokenizer.from(modelFolder: directory) + let effectiveDirectory = Self.remappingUnigramTokenizerClassIfNeeded(in: directory, modelId: modelId) + let t = try await AutoTokenizer.from(modelFolder: effectiveDirectory) return TransformersTokenizerBridge(t, modelId: modelId) } + + /// swift-transformers' `TokenizerModel.from(...)` picks a concrete tokenizer + /// implementation purely from the `tokenizer_class` name string (its + /// `knownTokenizers` table) — it never inspects `tokenizer.json`'s own + /// `model.type`. After stripping a `"Fast"` suffix, `"PreTrainedTokenizerFast"` + /// becomes `"PreTrainedTokenizer"`, which the table maps explicitly to + /// `BPETokenizer`. When the checkpoint's actual `tokenizer.json` is Unigram + /// (SentencePiece-style — common for models trained from scratch with a custom + /// vocab, e.g. several Japanese LLM projects that build their own vocab and + /// save via `AutoTokenizer`/`PreTrainedTokenizerFast` without a dedicated + /// subclass), there is no `merges` field and `BPETokenizer` hits a + /// `fatalError` instead of throwing (SwiftLM#155, reported by @kei-Optim). + /// + /// Workaround until the fix lands upstream in `huggingface/swift-transformers` + /// (`TokenizerModel.from` should consult `tokenizer.json`'s `model.type` when + /// `tokenizer_class` is generic/unknown): if `tokenizer.json` declares a + /// Unigram model and `tokenizer_config.json`'s `tokenizer_class` isn't already + /// one of the table's Unigram-mapped names, build a scratch copy of the + /// checkpoint directory — original files referenced via symlink, so the HF + /// cache is untouched — with `tokenizer_config.json`'s `tokenizer_class` + /// rewritten to `"XLMRobertaTokenizer"`, which resolves to `UnigramTokenizer`. + /// Its `init(tokenizerConfig:tokenizerData:addedTokens:)` doesn't depend on the + /// class name itself, so the rewrite is otherwise inert. + private static func remappingUnigramTokenizerClassIfNeeded( + in directory: URL, modelId: String + ) -> URL { + let tokenizerDataURL = directory.appendingPathComponent("tokenizer.json") + let tokenizerConfigURL = directory.appendingPathComponent("tokenizer_config.json") + guard + let tokenizerData = try? Data(contentsOf: tokenizerDataURL), + let tokenizerJSON = try? JSONSerialization.jsonObject(with: tokenizerData) as? [String: Any], + (tokenizerJSON["model"] as? [String: Any])?["type"] as? String == "Unigram", + let configData = try? Data(contentsOf: tokenizerConfigURL), + var configJSON = try? JSONSerialization.jsonObject(with: configData) as? [String: Any] + else { + return directory + } + + let unigramMappedClasses: Set = [ + "XLMRobertaTokenizer", "Xlm-RobertaTokenizer", "T5Tokenizer", + ] + let currentClass = (configJSON["tokenizer_class"] as? String)? + .replacingOccurrences(of: "Fast", with: "") + if let currentClass, unigramMappedClasses.contains(currentClass) { + return directory + } + + let originalClassDescription = (configJSON["tokenizer_class"] as? String) ?? "unset" + let scratchDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("swiftlm-unigram-tokenizer-\(UUID().uuidString)", isDirectory: true) + do { + try FileManager.default.createDirectory( + at: scratchDirectory, withIntermediateDirectories: true) + let contents = try FileManager.default.contentsOfDirectory( + at: directory, includingPropertiesForKeys: nil) + for fileURL in contents where fileURL.lastPathComponent != "tokenizer_config.json" { + try FileManager.default.createSymbolicLink( + at: scratchDirectory.appendingPathComponent(fileURL.lastPathComponent), + withDestinationURL: fileURL) + } + configJSON["tokenizer_class"] = "XLMRobertaTokenizer" + let rewritten = try JSONSerialization.data( + withJSONObject: configJSON, options: [.prettyPrinted]) + try rewritten.write(to: scratchDirectory.appendingPathComponent("tokenizer_config.json")) + } catch { + // Scratch-directory setup is best-effort — fall through to the original + // directory rather than fail the whole load over a workaround failing. + return directory + } + + print( + "[SwiftLM] \(modelId): tokenizer.json is Unigram but tokenizer_class is generic " + + "(\"\(originalClassDescription)\"); loading via a remapped tokenizer_class " + + "so it resolves to UnigramTokenizer instead of crashing in BPETokenizer.") + return scratchDirectory + } } /// A chat template that the Jinja parser rejected. diff --git a/scripts/make-test-fixtures.py b/scripts/make-test-fixtures.py index 2a6d8a5..3fddbea 100755 --- a/scripts/make-test-fixtures.py +++ b/scripts/make-test-fixtures.py @@ -77,6 +77,44 @@ def write_tokenizer(out): ) +def write_unigram_tokenizer(out): + """SwiftLM#155: a SentencePiece-style Unigram model.json, saved (as several real + Japanese-LLM checkpoints do) with the generic tokenizer_class "PreTrainedTokenizerFast" + rather than a class swift-transformers' knownTokenizers table maps to UnigramTokenizer + (XLMRobertaTokenizer, Xlm-RobertaTokenizer, T5Tokenizer). Before #155's fix, that + combination resolved to BPETokenizer, which fatalErrors on the missing "merges" field + instead of throwing. Same 288-token vocab as write_tokenizer, just a different model + algorithm and score instead of merges, so this drops in for build_dense's config/weights + unchanged. + """ + vocab = [(t, 0.0 if t == SPECIALS[0] else -1.0) for t in SPECIALS] + for ch in sorted(pre_tokenizers.ByteLevel.alphabet()): + vocab.append((ch, -2.0 - len(vocab) * 0.01)) + while len(vocab) < VOCAB: + vocab.append((f"<|unused{len(vocab)}|>", -10.0)) + + tok = Tokenizer(models.Unigram(vocab, 0, byte_fallback=False)) + tok.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=False) + tok.decoder = decoders.ByteLevel() + tok.post_processor = processors.ByteLevel(trim_offsets=False) + tok.add_special_tokens(SPECIALS) + tok.save(os.path.join(out, "tokenizer.json")) + + json.dump( + { + # The bug trigger: generic tokenizer_class alongside a Unigram model.json. + "tokenizer_class": "PreTrainedTokenizerFast", + "bos_token": "<|endoftext|>", + "eos_token": "<|im_end|>", + "pad_token": "", + "unk_token": "<|endoftext|>", + "chat_template": CHAT_TEMPLATE, + }, + open(os.path.join(out, "tokenizer_config.json"), "w"), + indent=2, + ) + + def rand(rng, *shape): return (rng.standard_normal(shape) * 0.02).astype(np.float16) @@ -352,12 +390,15 @@ def build_moe_nested(out): "kv-shared-absent": (build_gemma4_kv_shared, {"vestigial": False}), "kv-shared-present": (build_gemma4_kv_shared, {"vestigial": True}), "moe-nested": (build_moe_nested, {}), + "unigram-tokenizer": (build_dense, {}, write_unigram_tokenizer), } if __name__ == "__main__": os.makedirs(ROOT, exist_ok=True) total = 0 - for name, (fn, kwargs) in FIXTURES.items(): + for name, spec in FIXTURES.items(): + fn, kwargs, *rest = spec + tokenizer_writer = rest[0] if rest else write_tokenizer out = os.path.join(ROOT, name) # Clear this fixture's own directory, and only its own. Writing over an existing # one leaves behind files the current builder no longer emits — flip stray-shard @@ -370,7 +411,7 @@ def build_moe_nested(out): shutil.rmtree(out) os.makedirs(out) n = fn(out, **kwargs) - write_tokenizer(out) + tokenizer_writer(out) size = sum(os.path.getsize(os.path.join(out, f)) for f in os.listdir(out)) total += size print(f" {name:<20} {n:>3} tensors {size/1024:>7.1f} KB") diff --git a/tests/fixtures/unigram-tokenizer/config.json b/tests/fixtures/unigram-tokenizer/config.json new file mode 100644 index 0000000..5f3594c --- /dev/null +++ b/tests/fixtures/unigram-tokenizer/config.json @@ -0,0 +1,16 @@ +{ + "model_type": "qwen2", + "architectures": [ + "Qwen2ForCausalLM" + ], + "vocab_size": 288, + "hidden_size": 64, + "intermediate_size": 128, + "num_hidden_layers": 2, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "max_position_embeddings": 512, + "rms_norm_eps": 1e-06, + "rope_theta": 10000.0, + "tie_word_embeddings": false +} \ No newline at end of file diff --git a/tests/fixtures/unigram-tokenizer/model.safetensors b/tests/fixtures/unigram-tokenizer/model.safetensors new file mode 100644 index 0000000..6d98f75 Binary files /dev/null and b/tests/fixtures/unigram-tokenizer/model.safetensors differ diff --git a/tests/fixtures/unigram-tokenizer/tokenizer.json b/tests/fixtures/unigram-tokenizer/tokenizer.json new file mode 100644 index 0000000..a76b1a1 --- /dev/null +++ b/tests/fixtures/unigram-tokenizer/tokenizer.json @@ -0,0 +1,1221 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 0, + "content": "<|endoftext|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 1, + "content": "<|im_start|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 2, + "content": "<|im_end|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 3, + "content": "", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": true + }, + "post_processor": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": false, + "use_regex": true + }, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "model": { + "type": "Unigram", + "unk_id": 0, + "vocab": [ + [ + "<|endoftext|>", + 0.0 + ], + [ + "<|im_start|>", + -1.0 + ], + [ + "<|im_end|>", + -1.0 + ], + [ + "", + -1.0 + ], + [ + "!", + -2.04 + ], + [ + "\"", + -2.05 + ], + [ + "#", + -2.06 + ], + [ + "$", + -2.07 + ], + [ + "%", + -2.08 + ], + [ + "&", + -2.09 + ], + [ + "'", + -2.1 + ], + [ + "(", + -2.11 + ], + [ + ")", + -2.12 + ], + [ + "*", + -2.13 + ], + [ + "+", + -2.14 + ], + [ + ",", + -2.15 + ], + [ + "-", + -2.16 + ], + [ + ".", + -2.17 + ], + [ + "/", + -2.18 + ], + [ + "0", + -2.19 + ], + [ + "1", + -2.2 + ], + [ + "2", + -2.21 + ], + [ + "3", + -2.22 + ], + [ + "4", + -2.23 + ], + [ + "5", + -2.24 + ], + [ + "6", + -2.25 + ], + [ + "7", + -2.26 + ], + [ + "8", + -2.27 + ], + [ + "9", + -2.2800000000000002 + ], + [ + ":", + -2.29 + ], + [ + ";", + -2.3 + ], + [ + "<", + -2.31 + ], + [ + "=", + -2.32 + ], + [ + ">", + -2.33 + ], + [ + "?", + -2.34 + ], + [ + "@", + -2.35 + ], + [ + "A", + -2.36 + ], + [ + "B", + -2.37 + ], + [ + "C", + -2.38 + ], + [ + "D", + -2.39 + ], + [ + "E", + -2.4 + ], + [ + "F", + -2.41 + ], + [ + "G", + -2.42 + ], + [ + "H", + -2.43 + ], + [ + "I", + -2.44 + ], + [ + "J", + -2.45 + ], + [ + "K", + -2.46 + ], + [ + "L", + -2.47 + ], + [ + "M", + -2.48 + ], + [ + "N", + -2.49 + ], + [ + "O", + -2.5 + ], + [ + "P", + -2.51 + ], + [ + "Q", + -2.52 + ], + [ + "R", + -2.5300000000000002 + ], + [ + "S", + -2.54 + ], + [ + "T", + -2.55 + ], + [ + "U", + -2.56 + ], + [ + "V", + -2.5700000000000003 + ], + [ + "W", + -2.58 + ], + [ + "X", + -2.59 + ], + [ + "Y", + -2.6 + ], + [ + "Z", + -2.61 + ], + [ + "[", + -2.62 + ], + [ + "\\", + -2.63 + ], + [ + "]", + -2.64 + ], + [ + "^", + -2.65 + ], + [ + "_", + -2.66 + ], + [ + "`", + -2.67 + ], + [ + "a", + -2.68 + ], + [ + "b", + -2.69 + ], + [ + "c", + -2.7 + ], + [ + "d", + -2.71 + ], + [ + "e", + -2.7199999999999998 + ], + [ + "f", + -2.73 + ], + [ + "g", + -2.74 + ], + [ + "h", + -2.75 + ], + [ + "i", + -2.76 + ], + [ + "j", + -2.77 + ], + [ + "k", + -2.7800000000000002 + ], + [ + "l", + -2.79 + ], + [ + "m", + -2.8 + ], + [ + "n", + -2.81 + ], + [ + "o", + -2.8200000000000003 + ], + [ + "p", + -2.83 + ], + [ + "q", + -2.84 + ], + [ + "r", + -2.85 + ], + [ + "s", + -2.86 + ], + [ + "t", + -2.87 + ], + [ + "u", + -2.88 + ], + [ + "v", + -2.89 + ], + [ + "w", + -2.9 + ], + [ + "x", + -2.91 + ], + [ + "y", + -2.92 + ], + [ + "z", + -2.93 + ], + [ + "{", + -2.94 + ], + [ + "|", + -2.95 + ], + [ + "}", + -2.96 + ], + [ + "~", + -2.9699999999999998 + ], + [ + "¡", + -2.98 + ], + [ + "¢", + -2.99 + ], + [ + "£", + -3.0 + ], + [ + "¤", + -3.01 + ], + [ + "¥", + -3.02 + ], + [ + "¦", + -3.0300000000000002 + ], + [ + "§", + -3.04 + ], + [ + "¨", + -3.05 + ], + [ + "©", + -3.06 + ], + [ + "ª", + -3.0700000000000003 + ], + [ + "«", + -3.08 + ], + [ + "¬", + -3.09 + ], + [ + "®", + -3.1 + ], + [ + "¯", + -3.1100000000000003 + ], + [ + "°", + -3.12 + ], + [ + "±", + -3.13 + ], + [ + "²", + -3.14 + ], + [ + "³", + -3.1500000000000004 + ], + [ + "´", + -3.16 + ], + [ + "µ", + -3.17 + ], + [ + "¶", + -3.1799999999999997 + ], + [ + "·", + -3.19 + ], + [ + "¸", + -3.2 + ], + [ + "¹", + -3.21 + ], + [ + "º", + -3.2199999999999998 + ], + [ + "»", + -3.23 + ], + [ + "¼", + -3.24 + ], + [ + "½", + -3.25 + ], + [ + "¾", + -3.26 + ], + [ + "¿", + -3.27 + ], + [ + "À", + -3.2800000000000002 + ], + [ + "Á", + -3.29 + ], + [ + "Â", + -3.3 + ], + [ + "Ã", + -3.31 + ], + [ + "Ä", + -3.3200000000000003 + ], + [ + "Å", + -3.33 + ], + [ + "Æ", + -3.34 + ], + [ + "Ç", + -3.35 + ], + [ + "È", + -3.3600000000000003 + ], + [ + "É", + -3.37 + ], + [ + "Ê", + -3.38 + ], + [ + "Ë", + -3.39 + ], + [ + "Ì", + -3.4000000000000004 + ], + [ + "Í", + -3.41 + ], + [ + "Î", + -3.42 + ], + [ + "Ï", + -3.4299999999999997 + ], + [ + "Ð", + -3.44 + ], + [ + "Ñ", + -3.45 + ], + [ + "Ò", + -3.46 + ], + [ + "Ó", + -3.4699999999999998 + ], + [ + "Ô", + -3.48 + ], + [ + "Õ", + -3.49 + ], + [ + "Ö", + -3.5 + ], + [ + "×", + -3.51 + ], + [ + "Ø", + -3.52 + ], + [ + "Ù", + -3.5300000000000002 + ], + [ + "Ú", + -3.54 + ], + [ + "Û", + -3.55 + ], + [ + "Ü", + -3.56 + ], + [ + "Ý", + -3.5700000000000003 + ], + [ + "Þ", + -3.58 + ], + [ + "ß", + -3.59 + ], + [ + "à", + -3.6 + ], + [ + "á", + -3.6100000000000003 + ], + [ + "â", + -3.62 + ], + [ + "ã", + -3.63 + ], + [ + "ä", + -3.64 + ], + [ + "å", + -3.6500000000000004 + ], + [ + "æ", + -3.66 + ], + [ + "ç", + -3.67 + ], + [ + "è", + -3.6799999999999997 + ], + [ + "é", + -3.69 + ], + [ + "ê", + -3.7 + ], + [ + "ë", + -3.71 + ], + [ + "ì", + -3.7199999999999998 + ], + [ + "í", + -3.73 + ], + [ + "î", + -3.74 + ], + [ + "ï", + -3.75 + ], + [ + "ð", + -3.76 + ], + [ + "ñ", + -3.77 + ], + [ + "ò", + -3.7800000000000002 + ], + [ + "ó", + -3.79 + ], + [ + "ô", + -3.8 + ], + [ + "õ", + -3.81 + ], + [ + "ö", + -3.8200000000000003 + ], + [ + "÷", + -3.83 + ], + [ + "ø", + -3.84 + ], + [ + "ù", + -3.85 + ], + [ + "ú", + -3.8600000000000003 + ], + [ + "û", + -3.87 + ], + [ + "ü", + -3.88 + ], + [ + "ý", + -3.89 + ], + [ + "þ", + -3.9000000000000004 + ], + [ + "ÿ", + -3.91 + ], + [ + "Ā", + -3.92 + ], + [ + "ā", + -3.9299999999999997 + ], + [ + "Ă", + -3.94 + ], + [ + "ă", + -3.95 + ], + [ + "Ą", + -3.96 + ], + [ + "ą", + -3.9699999999999998 + ], + [ + "Ć", + -3.98 + ], + [ + "ć", + -3.99 + ], + [ + "Ĉ", + -4.0 + ], + [ + "ĉ", + -4.01 + ], + [ + "Ċ", + -4.02 + ], + [ + "ċ", + -4.03 + ], + [ + "Č", + -4.04 + ], + [ + "č", + -4.05 + ], + [ + "Ď", + -4.0600000000000005 + ], + [ + "ď", + -4.07 + ], + [ + "Đ", + -4.08 + ], + [ + "đ", + -4.09 + ], + [ + "Ē", + -4.1 + ], + [ + "ē", + -4.109999999999999 + ], + [ + "Ĕ", + -4.12 + ], + [ + "ĕ", + -4.13 + ], + [ + "Ė", + -4.140000000000001 + ], + [ + "ė", + -4.15 + ], + [ + "Ę", + -4.16 + ], + [ + "ę", + -4.17 + ], + [ + "Ě", + -4.18 + ], + [ + "ě", + -4.1899999999999995 + ], + [ + "Ĝ", + -4.2 + ], + [ + "ĝ", + -4.21 + ], + [ + "Ğ", + -4.220000000000001 + ], + [ + "ğ", + -4.23 + ], + [ + "Ġ", + -4.24 + ], + [ + "ġ", + -4.25 + ], + [ + "Ģ", + -4.26 + ], + [ + "ģ", + -4.27 + ], + [ + "Ĥ", + -4.28 + ], + [ + "ĥ", + -4.29 + ], + [ + "Ħ", + -4.300000000000001 + ], + [ + "ħ", + -4.3100000000000005 + ], + [ + "Ĩ", + -4.32 + ], + [ + "ĩ", + -4.33 + ], + [ + "Ī", + -4.34 + ], + [ + "ī", + -4.35 + ], + [ + "Ĭ", + -4.359999999999999 + ], + [ + "ĭ", + -4.37 + ], + [ + "Į", + -4.38 + ], + [ + "į", + -4.390000000000001 + ], + [ + "İ", + -4.4 + ], + [ + "ı", + -4.41 + ], + [ + "IJ", + -4.42 + ], + [ + "ij", + -4.43 + ], + [ + "Ĵ", + -4.4399999999999995 + ], + [ + "ĵ", + -4.45 + ], + [ + "Ķ", + -4.46 + ], + [ + "ķ", + -4.470000000000001 + ], + [ + "ĸ", + -4.48 + ], + [ + "Ĺ", + -4.49 + ], + [ + "ĺ", + -4.5 + ], + [ + "Ļ", + -4.51 + ], + [ + "ļ", + -4.52 + ], + [ + "Ľ", + -4.53 + ], + [ + "ľ", + -4.54 + ], + [ + "Ŀ", + -4.550000000000001 + ], + [ + "ŀ", + -4.5600000000000005 + ], + [ + "Ł", + -4.57 + ], + [ + "ł", + -4.58 + ], + [ + "Ń", + -4.59 + ], + [ + "<|unused260|>", + -10.0 + ], + [ + "<|unused261|>", + -10.0 + ], + [ + "<|unused262|>", + -10.0 + ], + [ + "<|unused263|>", + -10.0 + ], + [ + "<|unused264|>", + -10.0 + ], + [ + "<|unused265|>", + -10.0 + ], + [ + "<|unused266|>", + -10.0 + ], + [ + "<|unused267|>", + -10.0 + ], + [ + "<|unused268|>", + -10.0 + ], + [ + "<|unused269|>", + -10.0 + ], + [ + "<|unused270|>", + -10.0 + ], + [ + "<|unused271|>", + -10.0 + ], + [ + "<|unused272|>", + -10.0 + ], + [ + "<|unused273|>", + -10.0 + ], + [ + "<|unused274|>", + -10.0 + ], + [ + "<|unused275|>", + -10.0 + ], + [ + "<|unused276|>", + -10.0 + ], + [ + "<|unused277|>", + -10.0 + ], + [ + "<|unused278|>", + -10.0 + ], + [ + "<|unused279|>", + -10.0 + ], + [ + "<|unused280|>", + -10.0 + ], + [ + "<|unused281|>", + -10.0 + ], + [ + "<|unused282|>", + -10.0 + ], + [ + "<|unused283|>", + -10.0 + ], + [ + "<|unused284|>", + -10.0 + ], + [ + "<|unused285|>", + -10.0 + ], + [ + "<|unused286|>", + -10.0 + ], + [ + "<|unused287|>", + -10.0 + ] + ], + "byte_fallback": false + } +} \ No newline at end of file diff --git a/tests/fixtures/unigram-tokenizer/tokenizer_config.json b/tests/fixtures/unigram-tokenizer/tokenizer_config.json new file mode 100644 index 0000000..ee42569 --- /dev/null +++ b/tests/fixtures/unigram-tokenizer/tokenizer_config.json @@ -0,0 +1,8 @@ +{ + "tokenizer_class": "PreTrainedTokenizerFast", + "bos_token": "<|endoftext|>", + "eos_token": "<|im_end|>", + "pad_token": "", + "unk_token": "<|endoftext|>", + "chat_template": "{% for m in messages %}<|im_start|>{{ m['role'] }}\n{{ m['content'] }}<|im_end|>\n{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}" +} \ No newline at end of file diff --git a/tests/test-fixtures.sh b/tests/test-fixtures.sh index 632a587..e14547c 100755 --- a/tests/test-fixtures.sh +++ b/tests/test-fixtures.sh @@ -13,7 +13,11 @@ # moe-nested #112: expert count nested under text_config only, with a # decoy count under audio_config; plus the fused # experts.gate_up_proj split that sanitize performs +# unigram-tokenizer #155: a Unigram tokenizer.json saved under the generic +# tokenizer_class "PreTrainedTokenizerFast", which resolved to +# BPETokenizer and fatalError'd on the missing "merges" field # + # The output is gibberish by construction — the weights are random. A fixture passes # when the server loads it and produces *a* token, which is what exercises config # parsing, weight-key resolution, sanitisation and layer materialisation. @@ -105,7 +109,7 @@ sys.exit(0 if ok else 1)' 2>/dev/null; then } log "Binary: $BINARY" -for name in dense stray-shard kv-shared-absent kv-shared-present moe-nested; do +for name in dense stray-shard kv-shared-absent kv-shared-present moe-nested unigram-tokenizer; do log "Shape: $name" run_fixture "$name" done