feat(gguf): serve GLM-5.3-Flash GGUF checkpoints on glm5-next

python/freetoken/kernel/aot_models.py

@@ -281,7 +281,8 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int
         # KDA conv/recurrent state lives in the LinearStatePool, not paged KV.
         name="RedHatAI/GLM-5.3-Flash-NVFP4",
         architecture="Glm5NextForCausalLM",
-        arch_aliases=("Glm5NextForConditionalGeneration",),
+        # GGUF expert rows differ per layer and reach the fused copy as runtime sizes
+        arch_aliases=("Glm5NextForConditionalGeneration", "Glm5NextGGUFForCausalLM"),
         hidden_size=4096,
         kv_groups=(),
         top_k=8,

python/freetoken/layers/quantization/embedding/gguf.py

@@ -45,4 +45,4 @@ class GGUFEmbeddingMethod(EmbeddingMethod):
     def create_weights(self, layer: Any) -> None:
         (ggml_type,) = gguf_types(self.cfg.scheme)
         h = self.cfg.embedding_dim
-        layer.weight = torch.empty(self.cfg.num_embeddings + pad_rows(h), row_bytes(ggml_type, h), dtype=torch.uint8)
+        layer.weight = torch.empty(self.cfg.num_embeddings + pad_rows(ggml_type, h), row_bytes(ggml_type, h), dtype=torch.uint8)

python/freetoken/layers/quantization/ggml_types.py

@@ -44,9 +44,10 @@ def row_bytes(ggml_type: int, k: int) -> int:
     return k // block * size
 
 
-def pad_rows(k: int) -> int:
+def pad_rows(ggml_type: int, k: int) -> int:
     """Zero rows to append after a packed [rows, K] matrix so ggml's tail reads stay inside it."""
-    return 0 if k % MATRIX_ROW_PADDING == 0 else 1
+    tail = mmq_tail_bytes(ggml_type, k)
+    return 0 if tail == 0 else max(1, -(-tail // row_bytes(ggml_type, k)))
 
 
 def mmq_tail_bytes(ggml_type: int, k: int) -> int:

python/freetoken/layers/quantization/linear/gguf.py

@@ -54,7 +54,7 @@ def create_weights(self, layer: Any) -> None:
         k = g.in_features
         for i, s in enumerate(segs):
             if is_quantized(s.ggml_type):
-                setattr(layer, segment_param(i), torch.empty(s.rows + pad_rows(k), row_bytes(s.ggml_type, k), dtype=torch.uint8))
+                setattr(layer, segment_param(i), torch.empty(s.rows + pad_rows(s.ggml_type, k), row_bytes(s.ggml_type, k), dtype=torch.uint8))
             else:
                 setattr(layer, segment_param(i), torch.empty(s.rows, k))
         if g.scheme.weight.scale is not None:

python/freetoken/models/gguf/config.py

@@ -19,6 +19,9 @@
 GGUF_ARCH_TO_REGISTRY: dict[str, str] = {
     "gemma4": "Gemma4GGUFForCausalLM",
     "glm4moe": "Glm4MoeGGUFForCausalLM",
+    # glm5-next: the llama.cpp spelling; glm5next: unsloth's first release
+    "glm5-next": "Glm5NextGGUFForCausalLM",
+    "glm5next": "Glm5NextGGUFForCausalLM",
     "llama": "LlamaGGUFForCausalLM",
     "minimax-m2": "MiniMaxM2GGUFForCausalLM",
     "minimax-m3": "MiniMaxM3GGUFForCausalLM",

python/freetoken/models/gguf/experts.py

@@ -40,10 +40,10 @@ def _drop_page_cache(info: GgufTensorInfo) -> None:
         pass
 
 
-def _device_bank(shape: tuple[int, ...], k: int, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
+def _device_bank(shape: tuple[int, ...], ggml_type: int, k: int, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
     """An [E, rows, row] device bank; a zero row follows the last one when ggml's MMQ tiles read past it."""
     experts, rows, row = shape
-    flat = torch.empty(experts * rows + pad_rows(k), row, dtype=dtype, device=device)
+    flat = torch.empty(experts * rows + pad_rows(ggml_type, k), row, dtype=dtype, device=device)
     flat[experts * rows :].zero_()
     return flat[: experts * rows].view(shape)
 
@@ -101,8 +101,8 @@ def load_gguf_expert_banks(model_path: str, num_layers: int, *, first_layer: int
                 banks = {role: bank.tensor for role, bank in host.items()}
             else:
                 banks = {
-                    "gate_up": _device_bank(gate_up_shape, gate.ne[0], rows[0].dtype, device),
-                    "down": _device_bank(tuple(d.shape), down.ne[0], d.dtype, device),
+                    "gate_up": _device_bank(gate_up_shape, gate.ggml_type, gate.ne[0], rows[0].dtype, device),
+                    "down": _device_bank(tuple(d.shape), down.ggml_type, down.ne[0], d.dtype, device),
                 }
                 if scaled:
                     banks["global"] = torch.empty(experts, 4, dtype=torch.float32, device=device)

python/freetoken/models/gguf/tensors.py

@@ -85,7 +85,7 @@ def fused(prefix: str, infos: list[GgufTensorInfo], parts: list[torch.Tensor], k
         rows = parts[seg.first_part : seg.first_part + seg.num_parts]
         zeros = pad if i == len(segs) - 1 else 0
         if is_quantized(seg.ggml_type):
-            zeros += pad_rows(k)
+            zeros += pad_rows(seg.ggml_type, k)
             dtype = torch.uint8
         else:
             rows = [r.to(torch.bfloat16) for r in rows]

python/freetoken/models/gguf/tokenizer.py

@@ -23,7 +23,7 @@
 ))
 
 # Llama 3 and GLM pre-tokenizers, whose HF tokenizer and llama.cpp take a word that is one vocabulary token as is instead of merging it
-_IGNORE_MERGES_PRE = frozenset(("llama3", "llama-v3", "llama-bpe", "glm4"))
+_IGNORE_MERGES_PRE = frozenset(("llama3", "llama-v3", "llama-bpe", "glm4", "glm5"))
 
 
 def _add_bos(meta: dict) -> bool:
@@ -81,11 +81,12 @@ def load_gguf_tokenizer(model_path: str):
     meta = load_gguf_metadata(model_path)
     fast = _fix_added_tokens(fast, meta)
     pre = meta.get("tokenizer.ggml.pre") if meta.get("tokenizer.ggml.model") == "gpt2" else None
-    if pre == "glm4":
+    if pre in ("glm4", "glm5"):
+        # llama.cpp spells the same GLM pre-tokenizer "glm4" and "glm5" (llama-vocab.cpp maps both
+        # to CHATGLM4 with ignore_merges); transformers maps no split for either
         from tokenizers import Regex, pre_tokenizers
         from transformers.integrations.gguf.gguf_tokenizer_mapping import GGUF_PRE_TOKENIZER_SPLITS
 
-        # transformers maps no split for glm4, which the HF tokenizer and llama.cpp split like Llama 3
         fast.pre_tokenizer = pre_tokenizers.Sequence([
             pre_tokenizers.Split(Regex(GGUF_PRE_TOKENIZER_SPLITS["llama3"]), behavior="isolated"),
             pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),

python/freetoken/models/glm5_next/__init__.py

@@ -1,4 +1,5 @@
 from .config import VisionConfig, parse_config, parse_vision_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
 from .model import Glm5NextForCausalLM, Glm5NextForConditionalGeneration
 from .vision import Glm5NextVisionModel
 from .weight import iter_expert_pieces, iter_vision_weights, iter_weights, nvfp4_expert_spec
@@ -10,8 +11,11 @@
     "Glm5NextVisionModel",
     "VisionConfig",
     "parse_config",
+    "parse_gguf_config",
     "parse_vision_config",
     "iter_vision_weights",
+    "iter_gguf_weights",
     "iter_weights",
+    "to_gguf",
     "iter_expert_pieces",
 ]