From ba6409d4eff7cef297989685d0005231e6fb66a2 Mon Sep 17 00:00:00 2001 From: cazz Date: Mon, 22 Dec 2025 03:13:59 +0200 Subject: [PATCH 01/34] fix: correct assignment operator and method signature - fix tokenizer model_type assignment (was comparison) - fix cast_bias_weight call signature in embedding forward --- loader.py | 2 +- ops.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/loader.py b/loader.py index 7cefb11..191ce38 100644 --- a/loader.py +++ b/loader.py @@ -347,7 +347,7 @@ def gguf_tokenizer_loader(path, temb_shape): if get_field(reader, "tokenizer.ggml.model", str) == "t5": if temb_shape == (256384, 4096): # probably UMT5 - spm.trainer_spec.model_type == 1 # Unigram (do we have a T5 w/ BPE?) + spm.trainer_spec.model_type = 1 # Unigram (do we have a T5 w/ BPE?) else: raise NotImplementedError("Unknown model, can't set tokenizer!") else: diff --git a/ops.py b/ops.py index 88a352e..2ab4df8 100644 --- a/ops.py +++ b/ops.py @@ -253,7 +253,7 @@ def forward_ggml_cast_weights(self, input, out_dtype=None): output_dtype = out_dtype if self.weight.dtype == torch.float16 or self.weight.dtype == torch.bfloat16: out_dtype = None - weight, _bias = self.cast_bias_weight(self, device=input.device, dtype=out_dtype) + weight, _bias = self.cast_bias_weight(input, dtype=out_dtype) return torch.nn.functional.embedding( input, weight, self.padding_idx, self.max_norm, self.norm_type, self.scale_grad_by_freq, self.sparse ).to(dtype=output_dtype) From bc9af6ab86679bb12795a0f57e44b6bf5976109e Mon Sep 17 00:00:00 2001 From: cazz Date: Mon, 22 Dec 2025 03:14:10 +0200 Subject: [PATCH 02/34] refactor: improve path resolution and code quality - add resolve_full_path helper to centralize path lookup - add resolve_clip_path with smart fallback logic - fix mutable default argument in update_folder_names_and_paths - add FileNotFoundError with descriptive messages - apply pep8 formatting for consistency --- nodes.py | 196 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 142 insertions(+), 54 deletions(-) diff --git a/nodes.py b/nodes.py index 4683514..c85b9e5 100644 --- a/nodes.py +++ b/nodes.py @@ -17,21 +17,36 @@ from .loader import gguf_sd_loader, gguf_clip_loader from .dequant import is_quantized, is_torch_compatible -def update_folder_names_and_paths(key, targets=[]): + +def update_folder_names_and_paths(key, targets=None): + if targets is None: + targets = [] # check for existing key base = folder_paths.folder_names_and_paths.get(key, ([], {})) base = base[0] if isinstance(base[0], (list, set, tuple)) else [] # find base key & add w/ fallback, sanity check + warning - target = next((x for x in targets if x in folder_paths.folder_names_and_paths), targets[0]) + target = next( + (x for x in targets if x in folder_paths.folder_names_and_paths), targets[0] + ) orig, _ = folder_paths.folder_names_and_paths.get(target, ([], {})) folder_paths.folder_names_and_paths[key] = (orig or base, {".gguf"}) if base and base != orig: logging.warning(f"Unknown file list already present on key {key}: {base}") + +def resolve_full_path(name, keys): + for key in keys: + path = folder_paths.get_full_path(key, name) + if path: + return path + return None + + # Add a custom keys for files ending in .gguf update_folder_names_and_paths("unet_gguf", ["diffusion_models", "unet"]) update_folder_names_and_paths("clip_gguf", ["text_encoders", "clip"]) + class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): patch_on_device = False @@ -43,18 +58,26 @@ def patch_weight_to_device(self, key, device_to=None, inplace_update=False): patches = self.patches[key] if is_quantized(weight): out_weight = weight.to(device_to) - patches = move_patch_to_device(patches, self.load_device if self.patch_on_device else self.offload_device) + patches = move_patch_to_device( + patches, + self.load_device if self.patch_on_device else self.offload_device, + ) # TODO: do we ever have legitimate duplicate patches? (i.e. patch on top of patched weight) out_weight.patches = [(patches, key)] else: inplace_update = self.weight_inplace_update or inplace_update if key not in self.backup: - self.backup[key] = collections.namedtuple('Dimension', ['weight', 'inplace_update'])( - weight.to(device=self.offload_device, copy=inplace_update), inplace_update + self.backup[key] = collections.namedtuple( + "Dimension", ["weight", "inplace_update"] + )( + weight.to(device=self.offload_device, copy=inplace_update), + inplace_update, ) if device_to is not None: - temp_weight = comfy.model_management.cast_to_device(weight, device_to, torch.float32, copy=True) + temp_weight = comfy.model_management.cast_to_device( + weight, device_to, torch.float32, copy=True + ) else: temp_weight = weight.to(torch.float32, copy=True) @@ -75,14 +98,17 @@ def unpatch_model(self, device_to=None, unpatch_weights=True): if len(patches) > 0: p.patches = [] # TODO: Find another way to not unload after patches - return super().unpatch_model(device_to=device_to, unpatch_weights=unpatch_weights) - + return super().unpatch_model( + device_to=device_to, unpatch_weights=unpatch_weights + ) def pin_weight_to_device(self, key): - op_key = key.rsplit('.', 1)[0] + op_key = key.rsplit(".", 1)[0] if not self.mmap_released and op_key in self.named_modules_to_munmap: # TODO: possible to OOM, find better way to detach - self.named_modules_to_munmap[op_key].to(self.load_device).to(self.offload_device) + self.named_modules_to_munmap[op_key].to(self.load_device).to( + self.offload_device + ) del self.named_modules_to_munmap[op_key] super().pin_weight_to_device(key) @@ -129,9 +155,10 @@ def clone(self, *args, **kwargs): n.patch_on_device = getattr(self, "patch_on_device", False) n.mmap_released = getattr(self, "mmap_released", False) if src_cls != GGUFModelPatcher: - n.size = 0 # force recalc + n.size = 0 # force recalc return n + class UnetLoaderGGUF: @classmethod def INPUT_TYPES(s): @@ -147,7 +174,9 @@ def INPUT_TYPES(s): CATEGORY = "bootleg" TITLE = "Unet Loader (GGUF)" - def load_unet(self, unet_name, dequant_dtype=None, patch_dtype=None, patch_on_device=None): + def load_unet( + self, unet_name, dequant_dtype=None, patch_dtype=None, patch_on_device=None + ): ops = GGMLOps() if dequant_dtype in ("default", None): @@ -164,25 +193,29 @@ def load_unet(self, unet_name, dequant_dtype=None, patch_dtype=None, patch_on_de else: ops.Linear.patch_dtype = getattr(torch, patch_dtype) - # init model - unet_path = folder_paths.get_full_path("unet", unet_name) - sd, extra = gguf_sd_loader(unet_path) - - kwargs = {} - valid_params = inspect.signature(comfy.sd.load_diffusion_model_state_dict).parameters - if "metadata" in valid_params: - kwargs["metadata"] = extra.get("metadata", {}) - + # init model + unet_path = resolve_full_path(unet_name, ("unet_gguf", "unet")) + if unet_path is None: + raise FileNotFoundError(f"Unable to find UNet file: {unet_name}") + sd, extra = gguf_sd_loader(unet_path) + + kwargs = {} + valid_params = inspect.signature(comfy.sd.load_diffusion_model_state_dict).parameters + if "metadata" in valid_params: + kwargs["metadata"] = extra.get("metadata", {}) model = comfy.sd.load_diffusion_model_state_dict( sd, model_options={"custom_operations": ops}, **kwargs, ) if model is None: logging.error("ERROR UNSUPPORTED UNET {}".format(unet_path)) - raise RuntimeError("ERROR: Could not detect model type of: {}".format(unet_path)) + raise RuntimeError( + "ERROR: Could not detect model type of: {}".format(unet_path) + ) model = GGUFModelPatcher.clone(model) model.patch_on_device = patch_on_device return (model,) + class UnetLoaderGGUFAdvanced(UnetLoaderGGUF): @classmethod def INPUT_TYPES(s): @@ -190,13 +223,21 @@ def INPUT_TYPES(s): return { "required": { "unet_name": (unet_names,), - "dequant_dtype": (["default", "target", "float32", "float16", "bfloat16"], {"default": "default"}), - "patch_dtype": (["default", "target", "float32", "float16", "bfloat16"], {"default": "default"}), + "dequant_dtype": ( + ["default", "target", "float32", "float16", "bfloat16"], + {"default": "default"}, + ), + "patch_dtype": ( + ["default", "target", "float32", "float16", "bfloat16"], + {"default": "default"}, + ), "patch_on_device": ("BOOLEAN", {"default": False}), } } + TITLE = "Unet Loader (GGUF/Advanced)" + class CLIPLoaderGGUF: @classmethod def INPUT_TYPES(s): @@ -213,6 +254,15 @@ def INPUT_TYPES(s): CATEGORY = "bootleg" TITLE = "CLIPLoader (GGUF)" + @staticmethod + def resolve_clip_path(name): + keys = ( + ("clip_gguf", "clip") + if name.lower().endswith(".gguf") + else ("clip", "clip_gguf") + ) + return resolve_full_path(name, keys) + @classmethod def get_filename_list(s): files = [] @@ -227,34 +277,43 @@ def load_data(self, ckpt_paths): sd = gguf_clip_loader(p) else: sd = comfy.utils.load_torch_file(p, safe_load=True) - if "scaled_fp8" in sd: # NOTE: Scaled FP8 would require different custom ops, but only one can be active - raise NotImplementedError(f"Mixing scaled FP8 with GGUF is not supported! Use regular CLIP loader or switch model(s)\n({p})") + if ( + "scaled_fp8" in sd + ): # NOTE: Scaled FP8 would require different custom ops, but only one can be active + raise NotImplementedError( + f"Mixing scaled FP8 with GGUF is not supported! Use regular CLIP loader or switch model(s)\n({p})" + ) clip_data.append(sd) return clip_data def load_patcher(self, clip_paths, clip_type, clip_data): clip = comfy.sd.load_text_encoder_state_dicts( - clip_type = clip_type, - state_dicts = clip_data, - model_options = { + clip_type=clip_type, + state_dicts=clip_data, + model_options={ "custom_operations": GGMLOps, - "initial_device": comfy.model_management.text_encoder_offload_device() + "initial_device": comfy.model_management.text_encoder_offload_device(), }, - embedding_directory = folder_paths.get_folder_paths("embeddings"), + embedding_directory=folder_paths.get_folder_paths("embeddings"), ) clip.patcher = GGUFModelPatcher.clone(clip.patcher) return clip def load_clip(self, clip_name, type="stable_diffusion"): - clip_path = folder_paths.get_full_path("clip", clip_name) - clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) + clip_path = self.resolve_clip_path(clip_name) + if clip_path is None: + raise FileNotFoundError(f"Unable to find CLIP file: {clip_name}") + clip_type = getattr( + comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION + ) return (self.load_patcher([clip_path], clip_type, self.load_data([clip_path])),) + class DualCLIPLoaderGGUF(CLIPLoaderGGUF): @classmethod def INPUT_TYPES(s): base = nodes.DualCLIPLoader.INPUT_TYPES() - file_options = (s.get_filename_list(), ) + file_options = (s.get_filename_list(),) return { "required": { "clip_name1": file_options, @@ -266,16 +325,23 @@ def INPUT_TYPES(s): TITLE = "DualCLIPLoader (GGUF)" def load_clip(self, clip_name1, clip_name2, type): - clip_path1 = folder_paths.get_full_path("clip", clip_name1) - clip_path2 = folder_paths.get_full_path("clip", clip_name2) + clip_path1 = self.resolve_clip_path(clip_name1) + if clip_path1 is None: + raise FileNotFoundError(f"Unable to find CLIP file: {clip_name1}") + clip_path2 = self.resolve_clip_path(clip_name2) + if clip_path2 is None: + raise FileNotFoundError(f"Unable to find CLIP file: {clip_name2}") clip_paths = (clip_path1, clip_path2) - clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) + clip_type = getattr( + comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION + ) return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths)),) + class TripleCLIPLoaderGGUF(CLIPLoaderGGUF): @classmethod def INPUT_TYPES(s): - file_options = (s.get_filename_list(), ) + file_options = (s.get_filename_list(),) return { "required": { "clip_name1": file_options, @@ -287,37 +353,59 @@ def INPUT_TYPES(s): TITLE = "TripleCLIPLoader (GGUF)" def load_clip(self, clip_name1, clip_name2, clip_name3, type="sd3"): - clip_path1 = folder_paths.get_full_path("clip", clip_name1) - clip_path2 = folder_paths.get_full_path("clip", clip_name2) - clip_path3 = folder_paths.get_full_path("clip", clip_name3) + clip_path1 = self.resolve_clip_path(clip_name1) + if clip_path1 is None: + raise FileNotFoundError(f"Unable to find CLIP file: {clip_name1}") + clip_path2 = self.resolve_clip_path(clip_name2) + if clip_path2 is None: + raise FileNotFoundError(f"Unable to find CLIP file: {clip_name2}") + clip_path3 = self.resolve_clip_path(clip_name3) + if clip_path3 is None: + raise FileNotFoundError(f"Unable to find CLIP file: {clip_name3}") clip_paths = (clip_path1, clip_path2, clip_path3) - clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) + clip_type = getattr( + comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION + ) return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths)),) + class QuadrupleCLIPLoaderGGUF(CLIPLoaderGGUF): @classmethod def INPUT_TYPES(s): - file_options = (s.get_filename_list(), ) + file_options = (s.get_filename_list(),) return { "required": { - "clip_name1": file_options, - "clip_name2": file_options, - "clip_name3": file_options, - "clip_name4": file_options, + "clip_name1": file_options, + "clip_name2": file_options, + "clip_name3": file_options, + "clip_name4": file_options, + } } - } TITLE = "QuadrupleCLIPLoader (GGUF)" - def load_clip(self, clip_name1, clip_name2, clip_name3, clip_name4, type="stable_diffusion"): - clip_path1 = folder_paths.get_full_path("clip", clip_name1) - clip_path2 = folder_paths.get_full_path("clip", clip_name2) - clip_path3 = folder_paths.get_full_path("clip", clip_name3) - clip_path4 = folder_paths.get_full_path("clip", clip_name4) + def load_clip( + self, clip_name1, clip_name2, clip_name3, clip_name4, type="stable_diffusion" + ): + clip_path1 = self.resolve_clip_path(clip_name1) + if clip_path1 is None: + raise FileNotFoundError(f"Unable to find CLIP file: {clip_name1}") + clip_path2 = self.resolve_clip_path(clip_name2) + if clip_path2 is None: + raise FileNotFoundError(f"Unable to find CLIP file: {clip_name2}") + clip_path3 = self.resolve_clip_path(clip_name3) + if clip_path3 is None: + raise FileNotFoundError(f"Unable to find CLIP file: {clip_name3}") + clip_path4 = self.resolve_clip_path(clip_name4) + if clip_path4 is None: + raise FileNotFoundError(f"Unable to find CLIP file: {clip_name4}") clip_paths = (clip_path1, clip_path2, clip_path3, clip_path4) - clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) + clip_type = getattr( + comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION + ) return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths)),) + NODE_CLASS_MAPPINGS = { "UnetLoaderGGUF": UnetLoaderGGUF, "CLIPLoaderGGUF": CLIPLoaderGGUF, From a23a2dd2dce16f73ce2b37a30d94a67945eda4e8 Mon Sep 17 00:00:00 2001 From: cazz Date: Fri, 9 Jan 2026 20:10:58 +0200 Subject: [PATCH 03/34] feat(nodes): add dropdown configuration helpers for loaders Add _dropdown() and _ensure_dropdown() helper functions to provide consistent dropdown configuration format expected by ComfyUI frontend in newer builds. Update all GGUF loader classes (UnetLoaderGGUF, UnetLoaderGGUFAdvanced, CLIPLoaderGGUF, DualCLIPLoaderGGUF, TripleCLIPLoaderGGUF, QuadrupleCLIPLoaderGGUF) to use the new helpers, ensuring proper frontend dropdown formatting with default value support. --- nodes.py | 162 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 91 insertions(+), 71 deletions(-) diff --git a/nodes.py b/nodes.py index c85b9e5..3a40591 100644 --- a/nodes.py +++ b/nodes.py @@ -34,17 +34,37 @@ def update_folder_names_and_paths(key, targets=None): logging.warning(f"Unknown file list already present on key {key}: {base}") -def resolve_full_path(name, keys): +def resolve_full_path(name, keys): for key in keys: path = folder_paths.get_full_path(key, name) if path: return path - return None - - -# Add a custom keys for files ending in .gguf -update_folder_names_and_paths("unet_gguf", ["diffusion_models", "unet"]) -update_folder_names_and_paths("clip_gguf", ["text_encoders", "clip"]) + return None + + +# ComfyUI frontend expects a config dict for dropdowns in newer builds. +def _dropdown(values, default=None): + values = list(values) + if default is None: + default = values[0] if values else "" + return (values, {"default": default}) + + +def _ensure_dropdown(spec, default=None): + if isinstance(spec, tuple): + if len(spec) >= 2 and isinstance(spec[1], dict): + return spec + if len(spec) == 1 and isinstance(spec[0], (list, tuple)): + return _dropdown(spec[0], default) + return spec + if isinstance(spec, list): + return _dropdown(spec, default) + return spec + + +# Add a custom keys for files ending in .gguf +update_folder_names_and_paths("unet_gguf", ["diffusion_models", "unet"]) +update_folder_names_and_paths("clip_gguf", ["text_encoders", "clip"]) class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): @@ -159,15 +179,15 @@ def clone(self, *args, **kwargs): return n -class UnetLoaderGGUF: - @classmethod - def INPUT_TYPES(s): - unet_names = [x for x in folder_paths.get_filename_list("unet_gguf")] - return { - "required": { - "unet_name": (unet_names,), - } - } +class UnetLoaderGGUF: + @classmethod + def INPUT_TYPES(s): + unet_names = [x for x in folder_paths.get_filename_list("unet_gguf")] + return { + "required": { + "unet_name": _dropdown(unet_names), + } + } RETURN_TYPES = ("MODEL",) FUNCTION = "load_unet" @@ -216,16 +236,16 @@ def load_unet( return (model,) -class UnetLoaderGGUFAdvanced(UnetLoaderGGUF): - @classmethod - def INPUT_TYPES(s): - unet_names = [x for x in folder_paths.get_filename_list("unet_gguf")] - return { - "required": { - "unet_name": (unet_names,), - "dequant_dtype": ( - ["default", "target", "float32", "float16", "bfloat16"], - {"default": "default"}, +class UnetLoaderGGUFAdvanced(UnetLoaderGGUF): + @classmethod + def INPUT_TYPES(s): + unet_names = [x for x in folder_paths.get_filename_list("unet_gguf")] + return { + "required": { + "unet_name": _dropdown(unet_names), + "dequant_dtype": ( + ["default", "target", "float32", "float16", "bfloat16"], + {"default": "default"}, ), "patch_dtype": ( ["default", "target", "float32", "float16", "bfloat16"], @@ -238,16 +258,16 @@ def INPUT_TYPES(s): TITLE = "Unet Loader (GGUF/Advanced)" -class CLIPLoaderGGUF: - @classmethod - def INPUT_TYPES(s): - base = nodes.CLIPLoader.INPUT_TYPES() - return { - "required": { - "clip_name": (s.get_filename_list(),), - "type": base["required"]["type"], - } - } +class CLIPLoaderGGUF: + @classmethod + def INPUT_TYPES(s): + base = nodes.CLIPLoader.INPUT_TYPES() + return { + "required": { + "clip_name": _dropdown(s.get_filename_list()), + "type": _ensure_dropdown(base["required"]["type"]), + } + } RETURN_TYPES = ("CLIP",) FUNCTION = "load_clip" @@ -309,18 +329,18 @@ def load_clip(self, clip_name, type="stable_diffusion"): return (self.load_patcher([clip_path], clip_type, self.load_data([clip_path])),) -class DualCLIPLoaderGGUF(CLIPLoaderGGUF): - @classmethod - def INPUT_TYPES(s): - base = nodes.DualCLIPLoader.INPUT_TYPES() - file_options = (s.get_filename_list(),) - return { - "required": { - "clip_name1": file_options, - "clip_name2": file_options, - "type": base["required"]["type"], - } - } +class DualCLIPLoaderGGUF(CLIPLoaderGGUF): + @classmethod + def INPUT_TYPES(s): + base = nodes.DualCLIPLoader.INPUT_TYPES() + file_options = _dropdown(s.get_filename_list()) + return { + "required": { + "clip_name1": file_options, + "clip_name2": file_options, + "type": _ensure_dropdown(base["required"]["type"]), + } + } TITLE = "DualCLIPLoader (GGUF)" @@ -338,17 +358,17 @@ def load_clip(self, clip_name1, clip_name2, type): return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths)),) -class TripleCLIPLoaderGGUF(CLIPLoaderGGUF): - @classmethod - def INPUT_TYPES(s): - file_options = (s.get_filename_list(),) - return { - "required": { - "clip_name1": file_options, - "clip_name2": file_options, - "clip_name3": file_options, - } - } +class TripleCLIPLoaderGGUF(CLIPLoaderGGUF): + @classmethod + def INPUT_TYPES(s): + file_options = _dropdown(s.get_filename_list()) + return { + "required": { + "clip_name1": file_options, + "clip_name2": file_options, + "clip_name3": file_options, + } + } TITLE = "TripleCLIPLoader (GGUF)" @@ -369,18 +389,18 @@ def load_clip(self, clip_name1, clip_name2, clip_name3, type="sd3"): return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths)),) -class QuadrupleCLIPLoaderGGUF(CLIPLoaderGGUF): - @classmethod - def INPUT_TYPES(s): - file_options = (s.get_filename_list(),) - return { - "required": { - "clip_name1": file_options, - "clip_name2": file_options, - "clip_name3": file_options, - "clip_name4": file_options, - } - } +class QuadrupleCLIPLoaderGGUF(CLIPLoaderGGUF): + @classmethod + def INPUT_TYPES(s): + file_options = _dropdown(s.get_filename_list()) + return { + "required": { + "clip_name1": file_options, + "clip_name2": file_options, + "clip_name3": file_options, + "clip_name4": file_options, + } + } TITLE = "QuadrupleCLIPLoader (GGUF)" From 5cea0fa8b8027aafb90bd82ebdec599bb5c8aede Mon Sep 17 00:00:00 2001 From: Rattus Date: Sun, 1 Mar 2026 16:57:01 +1000 Subject: [PATCH 04/34] quant_ops: Implement GGUF as a QT Vibe code. To be reviewed. --- quant_ops.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 quant_ops.py diff --git a/quant_ops.py b/quant_ops.py new file mode 100644 index 0000000..856d381 --- /dev/null +++ b/quant_ops.py @@ -0,0 +1,83 @@ +# GGML QuantizedTensor support for dynamic VRAM loading +import gguf +import torch +from dataclasses import dataclass + +try: + from comfy_kitchen.tensor import ( + QuantizedTensor, + QuantizedLayout, + BaseLayoutParams, + register_layout_class, + ) + _CK_AVAILABLE = True +except ImportError: + _CK_AVAILABLE = False + + class QuantizedTensor: + pass + + class QuantizedLayout: + pass + + class BaseLayoutParams: + pass + + def register_layout_class(name, cls): + pass + +from .dequant import dequantize_functions, TORCH_COMPATIBLE_QTYPES, is_quantized + +if _CK_AVAILABLE: + @dataclass(frozen=True) + class GGMLLayoutParams(BaseLayoutParams): + tensor_type: int # gguf.GGMLQuantizationType stored as int + + class GGMLLayout(QuantizedLayout): + Params = GGMLLayoutParams + + @classmethod + def quantize(cls, tensor, **kwargs): + raise NotImplementedError("Quantization to GGML format is not supported") + + @classmethod + def dequantize(cls, qdata, params): + qtype = gguf.GGMLQuantizationType(params.tensor_type) + oshape = params.orig_shape + + if qtype in TORCH_COMPATIBLE_QTYPES: + return qdata.reshape(oshape).to(params.orig_dtype) + + if qtype not in dequantize_functions: + from tqdm import tqdm + tqdm.write(f"Falling back to numpy dequant for qtype: {qtype.name}") + new = gguf.quants.dequantize(qdata.cpu().numpy(), qtype) + return torch.from_numpy(new).reshape(oshape).to(device=qdata.device, dtype=params.orig_dtype) + + block_size, type_size = gguf.GGML_QUANT_SIZES[qtype] + raw = qdata.reshape(-1).view(torch.uint8) + n_blocks = raw.numel() // type_size + blocks = raw.reshape((n_blocks, type_size)) + blocks = dequantize_functions[qtype](blocks, block_size, type_size, None) + return blocks.reshape(oshape).to(params.orig_dtype) + + @classmethod + def get_plain_tensors(cls, qtensor): + return (qtensor._qdata,) + + @classmethod + def state_dict_tensors(cls, qdata, params): + return {"weight": qdata} + + register_layout_class("GGMLLayout", GGMLLayout) + + +def make_quantized(qdata, tensor_type, tensor_shape, orig_dtype=torch.float16): + """Construct a GGML QuantizedTensor from raw packed data.""" + params = GGMLLayoutParams( + scale=torch.ones((), dtype=torch.float32), + orig_dtype=orig_dtype, + orig_shape=tuple(tensor_shape), + tensor_type=tensor_type.value if not isinstance(tensor_type, int) else tensor_type, + ) + return QuantizedTensor(qdata, "GGMLLayout", params) From 1ef0118e743b2dd12e47525b53cf50cbbd62da56 Mon Sep 17 00:00:00 2001 From: Rattus Date: Sun, 1 Mar 2026 17:25:22 +1000 Subject: [PATCH 05/34] loader changes for Dynamic VRAM If in dynamic mode, load GGUF as a QT. --- loader.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/loader.py b/loader.py index 7cefb11..3f063f1 100644 --- a/loader.py +++ b/loader.py @@ -8,6 +8,7 @@ from .ops import GGMLTensor from .dequant import is_quantized, dequantize_tensor +from .quant_ops import make_quantized IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "hidream", "cosmos", "ltxv", "hyvid", "wan", "lumina2", "qwen_image"} TXT_ARCH_LIST = {"t5", "t5encoder", "llama", "qwen2vl", "qwen3", "qwen3vl", "gemma3"} @@ -67,7 +68,7 @@ def get_gguf_metadata(reader): continue return metadata -def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=False): +def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=False, dynamic=False): """ Read state dict as fake tensors """ @@ -134,9 +135,15 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F shape = shape[:-1] # add to state dict - if tensor.tensor_type in {gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16}: - torch_tensor = torch_tensor.view(*shape) - state_dict[sd_key] = GGMLTensor(torch_tensor, tensor_type=tensor.tensor_type, tensor_shape=shape) + if dynamic: + if tensor.tensor_type not in {gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16}: + state_dict[sd_key] = make_quantized(torch_tensor, tensor.tensor_type, shape) + else: + state_dict[sd_key] = torch_tensor.view(*shape) + else: + if tensor.tensor_type in {gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16}: + torch_tensor = torch_tensor.view(*shape) + state_dict[sd_key] = GGMLTensor(torch_tensor, tensor_type=tensor.tensor_type, tensor_shape=shape) # 1D tensors shouldn't be quantized, this is a fix for BF16 if len(shape) <= 1 and tensor.tensor_type == gguf.GGMLQuantizationType.BF16: From 6d95238cac21c3d779e3543e869aa4f47ef7eda2 Mon Sep 17 00:00:00 2001 From: Rattus Date: Sun, 1 Mar 2026 19:54:58 +1000 Subject: [PATCH 06/34] nodes: UNET: reconstructable ans support dynamic mode Refactor this to support the new reconstructability protocol in the comfy core. This is needed for DynamicVRAM (to support legacy demotion for fallbacks). Add the logic for dynamic_vram construction. This is also needed for worksplit multi-gpu branch where the model is deep-cloned via reconstruction to put the model on two parallel GPUs. --- nodes.py | 45 ++++++++++++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/nodes.py b/nodes.py index 4683514..a1b8eb9 100644 --- a/nodes.py +++ b/nodes.py @@ -11,6 +11,7 @@ import comfy.utils import comfy.model_patcher import comfy.model_management +import comfy.memory_management import folder_paths from .ops import GGMLOps, move_patch_to_device @@ -132,6 +133,34 @@ def clone(self, *args, **kwargs): n.size = 0 # force recalc return n + +def _clone_patcher_to_gguf(model_patcher): + if model_patcher.is_dynamic(): + return GGUFModelPatcherDynamic.clone(model_patcher) + else: + return GGUFModelPatcher.clone(model_patcher) + +def _load_gguf_unet(unet_path, ops, disable_dynamic=False): + dynamic = not disable_dynamic and comfy.memory_management.aimdo_enabled + sd, extra = gguf_sd_loader(unet_path, dynamic=dynamic) + + kwargs = {} + valid_params = inspect.signature(comfy.sd.load_diffusion_model_state_dict).parameters + if "metadata" in valid_params: + kwargs["metadata"] = extra.get("metadata", {}) + + model = comfy.sd.load_diffusion_model_state_dict( + sd, model_options={} if dynamic else { "custom_operations" : ops }, disable_dynamic=disable_dynamic, **kwargs, + ) + if model is None: + logging.error("ERROR UNSUPPORTED UNET {}".format(unet_path)) + raise RuntimeError("ERROR: Could not detect model type of: {}".format(unet_path)) + model = _clone_patcher_to_gguf(model) + + model.cached_patcher_init = (_load_gguf_unet, (unet_path, ops)) + + return model + class UnetLoaderGGUF: @classmethod def INPUT_TYPES(s): @@ -164,22 +193,8 @@ def load_unet(self, unet_name, dequant_dtype=None, patch_dtype=None, patch_on_de else: ops.Linear.patch_dtype = getattr(torch, patch_dtype) - # init model unet_path = folder_paths.get_full_path("unet", unet_name) - sd, extra = gguf_sd_loader(unet_path) - - kwargs = {} - valid_params = inspect.signature(comfy.sd.load_diffusion_model_state_dict).parameters - if "metadata" in valid_params: - kwargs["metadata"] = extra.get("metadata", {}) - - model = comfy.sd.load_diffusion_model_state_dict( - sd, model_options={"custom_operations": ops}, **kwargs, - ) - if model is None: - logging.error("ERROR UNSUPPORTED UNET {}".format(unet_path)) - raise RuntimeError("ERROR: Could not detect model type of: {}".format(unet_path)) - model = GGUFModelPatcher.clone(model) + model = _load_gguf_unet(unet_path, ops) model.patch_on_device = patch_on_device return (model,) From 68b2b05acb757ba52a2289157df0c410682c633f Mon Sep 17 00:00:00 2001 From: Rattus Date: Sun, 1 Mar 2026 19:54:58 +1000 Subject: [PATCH 07/34] nodes: CLIP: make reconstructable and support dynamic mode Refactor this to support the new reconstructability protocol in the comfy core. This is needed for DynamicVRAM (to support legacy demotion for fallbacks). Add the logic for dynamic_vram construction. This is also needed for worksplit multi-gpu branch where the model is deep-cloned via reconstruction to put the model on two parallel GPUs. --- nodes.py | 65 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/nodes.py b/nodes.py index a1b8eb9..f71f87f 100644 --- a/nodes.py +++ b/nodes.py @@ -161,6 +161,38 @@ def _load_gguf_unet(unet_path, ops, disable_dynamic=False): return model +def _load_gguf_clip_patcher(clip_paths, clip_type, disable_dynamic=False): + return _load_gguf_clip(clip_paths, clip_type, disable_dynamic=disable_dynamic).patcher + +def _load_gguf_clip(clip_paths, clip_type, disable_dynamic=False): + dynamic = not disable_dynamic and comfy.memory_management.aimdo_enabled + + clip_data = [] + for p in clip_paths: + if p.endswith(".gguf"): + sd = gguf_clip_loader(p) + else: + sd = comfy.utils.load_torch_file(p, safe_load=True) + if not dynamic and "scaled_fp8" in sd: # NOTE: Scaled FP8 would require different custom ops, but only one can be active + raise NotImplementedError(f"Mixing scaled FP8 with GGUF is not supported! Use regular CLIP loader or switch model(s)\n({p})") + clip_data.append(sd) + + model_options = {"initial_device": comfy.model_management.text_encoder_offload_device()} + if not dynamic: + model_options["custom_operations"] = GGMLOps + + clip = comfy.sd.load_text_encoder_state_dicts( + clip_type = clip_type, + state_dicts = clip_data, + model_options = model_options, + embedding_directory = folder_paths.get_folder_paths("embeddings"), + disable_dynamic = disable_dynamic, + ) + clip.patcher = _clone_patcher_to_gguf(clip.patcher) + + clip.patcher.cached_patcher_init = (_load_gguf_clip_patcher, (clip_paths, clip_type)) + return clip + class UnetLoaderGGUF: @classmethod def INPUT_TYPES(s): @@ -235,35 +267,10 @@ def get_filename_list(s): files += folder_paths.get_filename_list("clip_gguf") return sorted(files) - def load_data(self, ckpt_paths): - clip_data = [] - for p in ckpt_paths: - if p.endswith(".gguf"): - sd = gguf_clip_loader(p) - else: - sd = comfy.utils.load_torch_file(p, safe_load=True) - if "scaled_fp8" in sd: # NOTE: Scaled FP8 would require different custom ops, but only one can be active - raise NotImplementedError(f"Mixing scaled FP8 with GGUF is not supported! Use regular CLIP loader or switch model(s)\n({p})") - clip_data.append(sd) - return clip_data - - def load_patcher(self, clip_paths, clip_type, clip_data): - clip = comfy.sd.load_text_encoder_state_dicts( - clip_type = clip_type, - state_dicts = clip_data, - model_options = { - "custom_operations": GGMLOps, - "initial_device": comfy.model_management.text_encoder_offload_device() - }, - embedding_directory = folder_paths.get_folder_paths("embeddings"), - ) - clip.patcher = GGUFModelPatcher.clone(clip.patcher) - return clip - def load_clip(self, clip_name, type="stable_diffusion"): clip_path = folder_paths.get_full_path("clip", clip_name) clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) - return (self.load_patcher([clip_path], clip_type, self.load_data([clip_path])),) + return (_load_gguf_clip([clip_path], clip_type),) class DualCLIPLoaderGGUF(CLIPLoaderGGUF): @classmethod @@ -285,7 +292,7 @@ def load_clip(self, clip_name1, clip_name2, type): clip_path2 = folder_paths.get_full_path("clip", clip_name2) clip_paths = (clip_path1, clip_path2) clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) - return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths)),) + return (_load_gguf_clip(clip_paths, clip_type),) class TripleCLIPLoaderGGUF(CLIPLoaderGGUF): @classmethod @@ -307,7 +314,7 @@ def load_clip(self, clip_name1, clip_name2, clip_name3, type="sd3"): clip_path3 = folder_paths.get_full_path("clip", clip_name3) clip_paths = (clip_path1, clip_path2, clip_path3) clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) - return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths)),) + return (_load_gguf_clip(clip_paths, clip_type),) class QuadrupleCLIPLoaderGGUF(CLIPLoaderGGUF): @classmethod @@ -331,7 +338,7 @@ def load_clip(self, clip_name1, clip_name2, clip_name3, clip_name4, type="stable clip_path4 = folder_paths.get_full_path("clip", clip_name4) clip_paths = (clip_path1, clip_path2, clip_path3, clip_path4) clip_type = getattr(comfy.sd.CLIPType, type.upper(), comfy.sd.CLIPType.STABLE_DIFFUSION) - return (self.load_patcher(clip_paths, clip_type, self.load_data(clip_paths)),) + return (_load_gguf_clip(clip_paths, clip_type),) NODE_CLASS_MAPPINGS = { "UnetLoaderGGUF": UnetLoaderGGUF, From 5e905a1b1194d897c7f0c9f35289bd425b982975 Mon Sep 17 00:00:00 2001 From: Rattus Date: Sun, 1 Mar 2026 20:53:35 +1000 Subject: [PATCH 08/34] nodes: factor out clone() core of GGUFModelPatcher Factor this out to a helper and implement the new core reconstruction protocol. Consider the mmap_released flag 1:1 with the underlying model such that it moves with the base model in model_override. --- nodes.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/nodes.py b/nodes.py index f71f87f..b41cd81 100644 --- a/nodes.py +++ b/nodes.py @@ -33,6 +33,19 @@ def update_folder_names_and_paths(key, targets=[]): update_folder_names_and_paths("unet_gguf", ["diffusion_models", "unet"]) update_folder_names_and_paths("clip_gguf", ["text_encoders", "clip"]) +def _clone_as_gguf_model_patcher(self, *args, model_override=None, **kwargs): + if model_override is None: + model_override = self.get_clone_model_override() + mmap_released = model_override[2] if len(model_override) > 2 else False + src_cls = self.__class__ + self.__class__ = GGUFModelPatcher + n = comfy.model_patcher.ModelPatcher.clone(self, *args, model_override=model_override, **kwargs) + n.__class__ = GGUFModelPatcher + self.__class__ = src_cls + n.patch_on_device = getattr(self, "patch_on_device", False) + n.mmap_released = mmap_released + return n + class GGUFModelPatcher(comfy.model_patcher.ModelPatcher): patch_on_device = False @@ -90,6 +103,9 @@ def pin_weight_to_device(self, key): mmap_released = False named_modules_to_munmap = {} + def get_clone_model_override(self): + return (*super().get_clone_model_override(), self.mmap_released) + def load(self, *args, force_patch_weights=False, **kwargs): if not self.mmap_released: self.named_modules_to_munmap = dict(self.model.named_modules()) @@ -121,22 +137,19 @@ def load(self, *args, force_patch_weights=False, **kwargs): self.named_modules_to_munmap = {} def clone(self, *args, **kwargs): - src_cls = self.__class__ - self.__class__ = GGUFModelPatcher - n = super().clone(*args, **kwargs) - n.__class__ = GGUFModelPatcher - self.__class__ = src_cls - # GGUF specific clone values below - n.patch_on_device = getattr(self, "patch_on_device", False) - n.mmap_released = getattr(self, "mmap_released", False) - if src_cls != GGUFModelPatcher: + n = _clone_as_gguf_model_patcher(self, *args, **kwargs) + if self.__class__ != GGUFModelPatcher: n.size = 0 # force recalc return n def _clone_patcher_to_gguf(model_patcher): if model_patcher.is_dynamic(): - return GGUFModelPatcherDynamic.clone(model_patcher) + src_cls = model_patcher.__class__ + model_patcher.__class__ = GGUFModelPatcherDynamic + n = model_patcher.clone() + model_patcher.__class__ = src_cls + return n else: return GGUFModelPatcher.clone(model_patcher) From 4c7635179cf19dc451782f903397720d38220970 Mon Sep 17 00:00:00 2001 From: Rattus Date: Sun, 1 Mar 2026 22:03:53 +1000 Subject: [PATCH 09/34] GGUFModelPatcherDynamic --- nodes.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/nodes.py b/nodes.py index b41cd81..d5ab69f 100644 --- a/nodes.py +++ b/nodes.py @@ -142,6 +142,35 @@ def clone(self, *args, **kwargs): n.size = 0 # force recalc return n +class GGUFModelPatcherDynamic(comfy.model_patcher.ModelPatcherDynamic): + patch_on_device = False + + def load(self, *args, **kwargs): + super().load(*args, **kwargs) + # GGML can't requantize after LoRA - demote lowvram_function to weight_function + for n, m in self.model.named_modules(): + for param_key in ("weight", "bias"): + attr = param_key + "_lowvram_function" + fn = getattr(m, attr, None) + if fn is not None: + setattr(m, attr, None) + fns = getattr(m, param_key + "_function", []) + fns.append(fn) + setattr(m, param_key + "_function", fns) + if self.patch_on_device: + for key in self.patches: + self.patches[key] = move_patch_to_device(self.patches[key], self.load_device) + + def clone(self, disable_dynamic=False, model_override=None): + if disable_dynamic: + if model_override is None: + temp = self.cached_patcher_init[0](*self.cached_patcher_init[1], disable_dynamic=True) + model_override = temp.get_clone_model_override() + n = _clone_as_gguf_model_patcher(self, model_override=model_override) + return n + n = super().clone(disable_dynamic=disable_dynamic, model_override=model_override) + n.patch_on_device = self.patch_on_device + return n def _clone_patcher_to_gguf(model_patcher): if model_patcher.is_dynamic(): From 7de582af241f2073ed561171df8db5d5cc25f1a8 Mon Sep 17 00:00:00 2001 From: Octopus Date: Mon, 6 Apr 2026 23:30:11 +0800 Subject: [PATCH 10/34] feat: add torch dequantization for IQ1_S, IQ1_M, IQ2_XXS, IQ2_S, IQ3_XXS, IQ3_S Implements native PyTorch dequantization functions for lower IQ quant types, replacing the slow numpy fallback path for models quantized with these formats (e.g. Unsloth UD quants used as text encoders). All six new functions are verified against gguf.quants.dequantize() reference. --- dequant.py | 224 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) diff --git a/dequant.py b/dequant.py index 78f5f26..025c682 100644 --- a/dequant.py +++ b/dequant.py @@ -1,5 +1,6 @@ # (c) City96 || Apache-2.0 (apache.org/licenses/LICENSE-2.0) import gguf +import numpy as np import torch from tqdm import tqdm @@ -240,6 +241,24 @@ def dequantize_blocks_Q2_K(blocks, block_size, type_size, dtype=None): # IQ quants KVALUES = torch.tensor([-127, -104, -83, -65, -49, -35, -22, -10, 1, 13, 25, 38, 53, 69, 89, 113], dtype=torch.int8) +def _get_iq_grid(iq_cls): + iq_cls.init_grid() + return torch.from_numpy(np.array(iq_cls.grid).squeeze().copy()) + +def _get_iq_ksigns(iq_cls): + iq_cls.init_grid() + return torch.from_numpy(np.frombuffer(iq_cls.ksigns, dtype=np.uint8).copy()) + +from gguf.quants import IQ1_M as _IQ1_M, IQ1_S as _IQ1_S, IQ2_S as _IQ2_S, IQ2_XXS as _IQ2_XXS, IQ3_S as _IQ3_S, IQ3_XXS as _IQ3_XXS + +GRID_IQ3_S = _get_iq_grid(_IQ3_S) +GRID_IQ3_XXS = _get_iq_grid(_IQ3_XXS) +GRID_IQ2_S = _get_iq_grid(_IQ2_S) +GRID_IQ2_XXS = _get_iq_grid(_IQ2_XXS) +GRID_IQ1_S = _get_iq_grid(_IQ1_S) +_get_iq_grid(_IQ1_M) # IQ1_M uses the same grid as IQ1_S internally +KSIGNS_IQ2_XXS = _get_iq_ksigns(_IQ2_XXS) + def dequantize_blocks_IQ4_NL(blocks, block_size, type_size, dtype=None): n_blocks = blocks.shape[0] @@ -284,6 +303,205 @@ def dequantize_blocks_IQ4_XS(blocks, block_size, type_size, dtype=None): return (dl * qs).reshape((n_blocks, -1)) +def dequantize_blocks_IQ3_S(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + + d, qs, qh, signs, scales = split_block_dims(blocks, 2, 64, 8, 32) + d = d.view(torch.float16).to(dtype) + + scales = scales.view(torch.uint8) + scales = torch.stack([scales & 0xF, scales >> 4], dim=-1).reshape((n_blocks, 8)) + db = d * (1 + 2 * scales.to(dtype)) + db = db.reshape((n_blocks, 8, 1, 1)) + + shifts = torch.arange(8, device=d.device, dtype=torch.uint8).reshape((1, 1, 8)) + signs = (signs.unsqueeze(-1) >> shifts) & 1 + signs = torch.where( + signs == 0, + torch.ones(1, dtype=dtype, device=d.device), + torch.full((1,), -1.0, dtype=dtype, device=d.device), + ) + signs = signs.reshape((n_blocks, 8, 8, 4)) + + qh_bits = (qh.unsqueeze(-1) >> shifts) & 1 + qh_bits = qh_bits.reshape((n_blocks, 64)) + qs = qs.to(torch.int16) | (qh_bits.to(torch.int16) << 8) + + grid = GRID_IQ3_S.to(dtype=dtype, device=d.device) + grid_val = grid[qs.to(torch.long)].reshape((n_blocks, 8, 8, 4)) + + return (db * grid_val * signs).reshape((n_blocks, QK_K)) + +def dequantize_blocks_IQ3_XXS(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + + d, qs, scales, _ = split_block_dims(blocks, 2, 64, 32) + d = d.view(torch.float16).to(dtype) + + scales = scales.reshape((n_blocks, 8, 4)).to(torch.int32) + scales = scales[:, :, 0] | scales[:, :, 1] << 8 | scales[:, :, 2] << 16 | scales[:, :, 3] << 24 + + db = d * (0.5 + ((scales >> 28) & 0xF).to(dtype)) * 0.5 + db = db.reshape((n_blocks, 8, 1, 1)) + + shifts = torch.tensor([0, 7, 14, 21], device=d.device, dtype=torch.int32).reshape((1, 1, 4)) + sign_indices = (scales.reshape((n_blocks, 8, 1)) >> shifts) & 0x7F + + ksigns = KSIGNS_IQ2_XXS.to(d.device) + sign_bytes = ksigns[sign_indices.to(torch.long)] + + shifts_bits = torch.arange(8, device=d.device, dtype=torch.uint8).reshape((1, 1, 1, 8)) + signs = (sign_bytes.unsqueeze(-1) >> shifts_bits) & 1 + signs = torch.where( + signs == 0, + torch.ones(1, dtype=dtype, device=d.device), + torch.full((1,), -1.0, dtype=dtype, device=d.device), + ) + signs = signs.reshape((n_blocks, 8, 4, 8)) + + grid = GRID_IQ3_XXS.to(dtype=dtype, device=d.device) + grid_val = grid[qs.to(torch.long)].reshape((n_blocks, 8, 4, 8)) + + return (db * grid_val * signs).reshape((n_blocks, QK_K)) + +def dequantize_blocks_IQ2_S(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + + d, qs, signs, qh, scales = split_block_dims(blocks, 2, 32, 32, 8) + d = d.view(torch.float16).to(dtype) + + scales = scales.view(torch.uint8) + scales = torch.stack([scales & 0xF, scales >> 4], dim=-1).reshape((n_blocks, 16)) + db = d * (0.5 + scales.to(dtype)) * 0.25 + db = db.reshape((n_blocks, 16, 1, 1)) + + shifts = torch.arange(8, device=d.device, dtype=torch.uint8).reshape((1, 1, 8)) + signs = (signs.unsqueeze(-1) >> shifts) & 1 + signs = torch.where( + signs == 0, + torch.ones(1, dtype=dtype, device=d.device), + torch.full((1,), -1.0, dtype=dtype, device=d.device), + ) + signs = signs.reshape((n_blocks, 16, 2, 8)) + + qh_shifts = torch.tensor([0, 2, 4, 6], device=d.device, dtype=torch.uint8).reshape((1, 1, 4)) + qh_bits = (qh.view(torch.uint8).reshape((n_blocks, 8, 1)) >> qh_shifts) & 3 + qh_bits = qh_bits.reshape((n_blocks, 32)) + + qs = qs.view(torch.uint8).to(torch.int32) + indices = qs | (qh_bits.to(torch.int32) << 8) + + grid = GRID_IQ2_S.to(dtype=dtype, device=d.device) + grid_val = grid[indices.to(torch.long)].reshape((n_blocks, 16, 2, 8)) + + return (db * grid_val * signs).reshape((n_blocks, QK_K)) + +def dequantize_blocks_IQ2_XXS(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + + d, qs = split_block_dims(blocks, 2) + d = d.view(torch.float16).to(dtype) + + u32 = qs.reshape((n_blocks, 16, 4)).to(torch.int32) + u32 = u32[:, :, 0] | (u32[:, :, 1] << 8) | (u32[:, :, 2] << 16) | (u32[:, :, 3] << 24) + u32 = u32.reshape((n_blocks, 8, 2)) + + q0 = u32[:, :, 0] # grid indices + q1 = u32[:, :, 1] # scales and signs + + db = d * (0.5 + ((q1 >> 28) & 0xF).to(dtype)) * 0.25 + db = db.reshape((n_blocks, 8, 1, 1)) + + shifts = torch.tensor([0, 7, 14, 21], device=d.device, dtype=torch.int32).reshape((1, 1, 4)) + sign_indices = (q1.unsqueeze(-1) >> shifts) & 0x7F + + ksigns = KSIGNS_IQ2_XXS.to(d.device) + sign_bytes = ksigns[sign_indices.to(torch.long)] + + shifts_bits = torch.arange(8, device=d.device, dtype=torch.uint8).reshape((1, 1, 1, 8)) + signs = (sign_bytes.unsqueeze(-1) >> shifts_bits) & 1 + signs = torch.where( + signs == 0, + torch.ones(1, dtype=dtype, device=d.device), + torch.full((1,), -1.0, dtype=dtype, device=d.device), + ) + signs = signs.reshape((n_blocks, 8, 4, 8)) + + indices = q0.contiguous().view(torch.uint8) + grid = GRID_IQ2_XXS.to(dtype=dtype, device=d.device) + grid_val = grid[indices.to(torch.long)].reshape((n_blocks, 8, 4, 8)) + + return (db * grid_val * signs).reshape((n_blocks, QK_K)) + +def dequantize_blocks_IQ1_M(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + + qs, qh, scales = split_block_dims(blocks, 32, 16) + + scales_u16 = scales.reshape((n_blocks, 4, 2)).to(torch.int32) + scales_u16 = scales_u16[:, :, 0] | (scales_u16[:, :, 1] << 8) + + d_bits = ( + ((scales_u16[:, 0] & 0xF000) >> 12) + | ((scales_u16[:, 1] & 0xF000) >> 8) + | ((scales_u16[:, 2] & 0xF000) >> 4) + | (scales_u16[:, 3] & 0xF000) + ) + d = d_bits.to(torch.int16).view(torch.float16).to(dtype).reshape((n_blocks, 1)) + + sub_shifts = torch.tensor([0, 3, 6, 9], device=d.device, dtype=torch.int32).reshape((1, 1, 4)) + sub_scales = (scales_u16.reshape((n_blocks, 4, 1)) >> sub_shifts) & 7 + dl = d.reshape((n_blocks, 1, 1)) * (2 * sub_scales.to(dtype) + 1) + dl = dl.reshape((n_blocks, 8, 2, 1, 1)) + + qh_bytes = qh.to(torch.int32) + qh_shifts = torch.tensor([0, 4], device=d.device, dtype=torch.int32).reshape((1, 1, 2)) + qh_unpacked = (qh_bytes.reshape((n_blocks, 16, 1)) >> qh_shifts).reshape((n_blocks, 32)) + + delta = torch.where( + (qh_unpacked & 8) == 0, + torch.full((1,), 0.125, dtype=dtype, device=d.device), + torch.full((1,), -0.125, dtype=dtype, device=d.device), + ).reshape((n_blocks, 8, 2, 2, 1)) + + qh_bits = qh_unpacked & 7 + qs = qs.to(torch.int32) + indices = qs | (qh_bits << 8) + + grid = GRID_IQ1_S.to(dtype=dtype, device=d.device) + grid_val = grid[indices.to(torch.long)].reshape((n_blocks, 8, 2, 2, 8)) + + return (dl * (grid_val + delta)).reshape((n_blocks, QK_K)) + +def dequantize_blocks_IQ1_S(blocks, block_size, type_size, dtype=None): + n_blocks = blocks.shape[0] + + d, qs, qh = split_block_dims(blocks, 2, 32) + d = d.view(torch.float16).to(dtype) + + qh = qh.view(torch.int16).to(torch.int32) & 0xFFFF + + dl = d * (2 * ((qh >> 12) & 7).to(dtype) + 1) + delta = torch.where( + (qh & 0x8000) == 0, + torch.full((1,), 0.125, dtype=dtype, device=d.device), + torch.full((1,), -0.125, dtype=dtype, device=d.device), + ) + + shifts = torch.tensor([0, 3, 6, 9], device=d.device, dtype=torch.int32).reshape((1, 1, 4)) + qh_bits = (qh.reshape((n_blocks, 8, 1)) >> shifts) & 7 + + qs = qs.view(torch.uint8).to(torch.int32).reshape((n_blocks, 8, 4)) + indices = qs | (qh_bits << 8) + + grid = GRID_IQ1_S.to(dtype=dtype, device=d.device) + grid_val = grid[indices.to(torch.long)].reshape((n_blocks, 8, 4, 8)) + + dl = dl.reshape((n_blocks, 8, 1, 1)) + delta = delta.reshape((n_blocks, 8, 1, 1)) + + return (dl * (grid_val + delta)).reshape((n_blocks, QK_K)) + dequantize_functions = { gguf.GGMLQuantizationType.BF16: dequantize_blocks_BF16, gguf.GGMLQuantizationType.Q8_0: dequantize_blocks_Q8_0, @@ -298,4 +516,10 @@ def dequantize_blocks_IQ4_XS(blocks, block_size, type_size, dtype=None): gguf.GGMLQuantizationType.Q2_K: dequantize_blocks_Q2_K, gguf.GGMLQuantizationType.IQ4_NL: dequantize_blocks_IQ4_NL, gguf.GGMLQuantizationType.IQ4_XS: dequantize_blocks_IQ4_XS, + gguf.GGMLQuantizationType.IQ3_S: dequantize_blocks_IQ3_S, + gguf.GGMLQuantizationType.IQ3_XXS: dequantize_blocks_IQ3_XXS, + gguf.GGMLQuantizationType.IQ2_S: dequantize_blocks_IQ2_S, + gguf.GGMLQuantizationType.IQ2_XXS: dequantize_blocks_IQ2_XXS, + gguf.GGMLQuantizationType.IQ1_M: dequantize_blocks_IQ1_M, + gguf.GGMLQuantizationType.IQ1_S: dequantize_blocks_IQ1_S, } From effce96e0dd1f301830253f6688eeb0f7fd43900 Mon Sep 17 00:00:00 2001 From: Rattus Date: Thu, 21 May 2026 23:04:04 +1000 Subject: [PATCH 11/34] nodes: fix gguf loading --- loader.py | 4 ++-- nodes.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/loader.py b/loader.py index 3f063f1..730a908 100644 --- a/loader.py +++ b/loader.py @@ -474,8 +474,8 @@ def gguf_gemma3_tokenizer_loader(path): del reader return torch.ByteTensor(list(spm.SerializeToString())) -def gguf_clip_loader(path): - sd, extra = gguf_sd_loader(path, is_text_model=True) +def gguf_clip_loader(path, dynamic=False): + sd, extra = gguf_sd_loader(path, is_text_model=True, dynamic=dynamic) arch = extra.get("arch_str", None) if arch in {"t5", "t5encoder"}: temb_key = "token_embd.weight" diff --git a/nodes.py b/nodes.py index d5ab69f..28bd645 100644 --- a/nodes.py +++ b/nodes.py @@ -212,7 +212,7 @@ def _load_gguf_clip(clip_paths, clip_type, disable_dynamic=False): clip_data = [] for p in clip_paths: if p.endswith(".gguf"): - sd = gguf_clip_loader(p) + sd = gguf_clip_loader(p, dynamic=dynamic) else: sd = comfy.utils.load_torch_file(p, safe_load=True) if not dynamic and "scaled_fp8" in sd: # NOTE: Scaled FP8 would require different custom ops, but only one can be active @@ -390,4 +390,3 @@ def load_clip(self, clip_name1, clip_name2, clip_name3, clip_name4, type="stable "QuadrupleCLIPLoaderGGUF": QuadrupleCLIPLoaderGGUF, "UnetLoaderGGUFAdvanced": UnetLoaderGGUFAdvanced, } - From 5c4b0487d6c82164991a3718406a43e32b485657 Mon Sep 17 00:00:00 2001 From: Danrisi Date: Fri, 12 Jun 2026 04:56:08 +0300 Subject: [PATCH 12/34] Add Ideogram 4 architecture support for GGUF loading Co-authored-by: Cursor --- loader.py | 2 +- tools/convert.py | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/loader.py b/loader.py index 7cefb11..a368236 100644 --- a/loader.py +++ b/loader.py @@ -9,7 +9,7 @@ from .ops import GGMLTensor from .dequant import is_quantized, dequantize_tensor -IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "hidream", "cosmos", "ltxv", "hyvid", "wan", "lumina2", "qwen_image"} +IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "hidream", "cosmos", "ltxv", "hyvid", "wan", "lumina2", "qwen_image", "ideogram4"} TXT_ARCH_LIST = {"t5", "t5encoder", "llama", "qwen2vl", "qwen3", "qwen3vl", "gemma3"} VIS_TYPE_LIST = {"clip-vision", "mmproj"} diff --git a/tools/convert.py b/tools/convert.py index 5029c87..4033377 100644 --- a/tools/convert.py +++ b/tools/convert.py @@ -145,8 +145,15 @@ class ModelLumina2(ModelTemplate): ("cap_embedder.1.weight", "context_refiner.0.attention.qkv.weight") ] +class ModelIdeogram4(ModelTemplate): + arch = "ideogram4" + keys_detect = [ + ("embed_image_indicator.weight", "llm_cond_proj.weight", "layers.0.adaln_modulation.weight"), + ("embed_image_indicator.weight", "input_proj.weight", "adaln_proj.weight"), + ] + arch_list = [ModelFlux, ModelSD3, ModelAura, ModelHiDream, CosmosPredict2, - ModelLTXV, ModelHyVid, ModelWan, ModelSDXL, ModelSD1, ModelLumina2] + ModelLTXV, ModelHyVid, ModelWan, ModelSDXL, ModelSD1, ModelLumina2, ModelIdeogram4] def is_model_arch(model, state_dict): # check if model is correct From 3224f5cd81a1dd2f4ca6b1200247726ef41d1c96 Mon Sep 17 00:00:00 2001 From: blepping Date: Tue, 16 Jun 2026 14:49:24 -0600 Subject: [PATCH 13/34] Handle WeightAdapters in ops.move_patch_to_device --- ops.py | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/ops.py b/ops.py index 88a352e..d5d920c 100644 --- a/ops.py +++ b/ops.py @@ -8,6 +8,11 @@ import comfy.model_management from .dequant import dequantize_tensor, is_quantized +try: + import comfy.weight_adapter as wadapter +except (ImportError, ModuleNotFoundError): + wadapter = None + def chained_hasattr(obj, chained_attr): probe = obj for attr in chained_attr.split('.'): @@ -270,12 +275,28 @@ def forward_ggml_cast_weights(self, input): weight, bias = self.cast_bias_weight(input) return torch.nn.functional.group_norm(input, self.num_groups, weight, bias, self.eps) -def move_patch_to_device(item, device): - if isinstance(item, torch.Tensor): - return item.to(device, non_blocking=True) - elif isinstance(item, tuple): - return tuple(move_patch_to_device(x, device) for x in item) - elif isinstance(item, list): - return [move_patch_to_device(x, device) for x in item] - else: +def move_patch_to_device(item, device, *, dtype=None): + if device is None: return item + if isinstance(item, torch.Tensor): + return comfy.model_management.cast_to_device(item, device, dtype, copy=True) + if isinstance(item, (tuple, list)): + return item.__class__( + move_patch_to_device(seqitem, device, dtype=dtype) for seqitem in item + ) + if ( + wadapter is not None + and isinstance(item, wadapter.WeightAdapterBase) + and hasattr(item, "loaded_keys") + and isinstance(getattr(item, "weights", None), (tuple, list)) + ): + return item.__class__( + item.loaded_keys, + item.weights.__class__( + wi + if not isinstance(wi, torch.Tensor) + else comfy.model_management.cast_to_device(wi, device, dtype, copy=True) + for wi in item.weights + ), + ) + return item From 7c3cee5deb6fa6ae53c247b8ba0f9564ef00fb19 Mon Sep 17 00:00:00 2001 From: m8rr Date: Thu, 18 Jun 2026 03:47:55 +0900 Subject: [PATCH 14/34] Apply torch.compile to dequant, up to 25% speedup depending on conditions --- quant_ops.py | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/quant_ops.py b/quant_ops.py index 856d381..b09ca77 100644 --- a/quant_ops.py +++ b/quant_ops.py @@ -28,6 +28,17 @@ def register_layout_class(name, cls): from .dequant import dequantize_functions, TORCH_COMPATIBLE_QTYPES, is_quantized +HAS_COMPILE = hasattr(torch, 'compile') +def conditional_compile(*args, **kwargs): + def decorator(func): + if HAS_COMPILE: + try: + return torch.compile(func, *args, **kwargs) + except Exception: + return func + return func + return decorator + if _CK_AVAILABLE: @dataclass(frozen=True) class GGMLLayoutParams(BaseLayoutParams): @@ -40,6 +51,14 @@ class GGMLLayout(QuantizedLayout): def quantize(cls, tensor, **kwargs): raise NotImplementedError("Quantization to GGML format is not supported") + + @staticmethod + @conditional_compile(fullgraph=True) + def _compiled_core_dequantize(qdata_raw, qtype, block_size, type_size, dequant_func): + n_blocks = qdata_raw.numel() // type_size + blocks = qdata_raw.reshape((n_blocks, type_size)) + return dequant_func(blocks, block_size, type_size, None) + @classmethod def dequantize(cls, qdata, params): qtype = gguf.GGMLQuantizationType(params.tensor_type) @@ -56,9 +75,11 @@ def dequantize(cls, qdata, params): block_size, type_size = gguf.GGML_QUANT_SIZES[qtype] raw = qdata.reshape(-1).view(torch.uint8) - n_blocks = raw.numel() // type_size - blocks = raw.reshape((n_blocks, type_size)) - blocks = dequantize_functions[qtype](blocks, block_size, type_size, None) + + blocks = cls._compiled_core_dequantize( + raw, qtype, block_size, type_size, dequantize_functions[qtype] + ) + return blocks.reshape(oshape).to(params.orig_dtype) @classmethod From a7c141b6db0a550d125c835a483a9c04ff7bcc5b Mon Sep 17 00:00:00 2001 From: m8rr Date: Thu, 18 Jun 2026 20:07:08 +0900 Subject: [PATCH 15/34] refactor: Apply torch.compile to dequant --- quant_ops.py | 40 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/quant_ops.py b/quant_ops.py index b09ca77..53e84dd 100644 --- a/quant_ops.py +++ b/quant_ops.py @@ -29,15 +29,7 @@ def register_layout_class(name, cls): from .dequant import dequantize_functions, TORCH_COMPATIBLE_QTYPES, is_quantized HAS_COMPILE = hasattr(torch, 'compile') -def conditional_compile(*args, **kwargs): - def decorator(func): - if HAS_COMPILE: - try: - return torch.compile(func, *args, **kwargs) - except Exception: - return func - return func - return decorator +COMPILED_DEQUANT_FUNCTIONS = {} if _CK_AVAILABLE: @dataclass(frozen=True) @@ -51,14 +43,6 @@ class GGMLLayout(QuantizedLayout): def quantize(cls, tensor, **kwargs): raise NotImplementedError("Quantization to GGML format is not supported") - - @staticmethod - @conditional_compile(fullgraph=True) - def _compiled_core_dequantize(qdata_raw, qtype, block_size, type_size, dequant_func): - n_blocks = qdata_raw.numel() // type_size - blocks = qdata_raw.reshape((n_blocks, type_size)) - return dequant_func(blocks, block_size, type_size, None) - @classmethod def dequantize(cls, qdata, params): qtype = gguf.GGMLQuantizationType(params.tensor_type) @@ -75,11 +59,11 @@ def dequantize(cls, qdata, params): block_size, type_size = gguf.GGML_QUANT_SIZES[qtype] raw = qdata.reshape(-1).view(torch.uint8) + n_blocks = raw.numel() // type_size + blocks = raw.reshape((n_blocks, type_size)) - blocks = cls._compiled_core_dequantize( - raw, qtype, block_size, type_size, dequantize_functions[qtype] - ) - + fn = get_compiled(qtype, dequantize_functions[qtype]) + blocks = fn(blocks, block_size, type_size, None) return blocks.reshape(oshape).to(params.orig_dtype) @classmethod @@ -92,6 +76,20 @@ def state_dict_tensors(cls, qdata, params): register_layout_class("GGMLLayout", GGMLLayout) +def get_compiled(qtype, raw_func): + if qtype in COMPILED_DEQUANT_FUNCTIONS: + return COMPILED_DEQUANT_FUNCTIONS[qtype] + + if HAS_COMPILE: + try: + compiled = torch.compile(raw_func, fullgraph=True) + except Exception: + compiled = raw_func + else: + compiled = raw_func + + COMPILED_DEQUANT_FUNCTIONS[qtype] = compiled + return compiled def make_quantized(qdata, tensor_type, tensor_shape, orig_dtype=torch.float16): """Construct a GGML QuantizedTensor from raw packed data.""" From b3abe74b24901774682b951334f95a91d53bd786 Mon Sep 17 00:00:00 2001 From: m8rr Date: Fri, 19 Jun 2026 15:32:29 +0900 Subject: [PATCH 16/34] cherry-pick BF16 fix from commit a5d4ce5 --- loader.py | 50 +++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/loader.py b/loader.py index b83406c..da4d7fc 100644 --- a/loader.py +++ b/loader.py @@ -14,6 +14,36 @@ TXT_ARCH_LIST = {"t5", "t5encoder", "llama", "qwen2vl", "qwen3", "qwen3vl", "gemma3"} VIS_TYPE_LIST = {"clip-vision", "mmproj"} +def device_supports_bf16(): + """ + Return True if the active torch device can run BF16 efficiently. Some + consumer Ampere GPUs report BF16 support through PyTorch/Comfy but route + model execution through FP32 manual casts, which is much slower. Keep this + stricter than Comfy's generic helper for GGUF tensor loading. + """ + try: + import comfy.model_management + device = torch.device(comfy.model_management.get_torch_device()) + if device.type != "cuda": + return comfy.model_management.should_use_bf16(device) + + index = device.index if device.index is not None else torch.cuda.current_device() + major, minor = torch.cuda.get_device_capability(index) + name = torch.cuda.get_device_name(index).lower() + + # Native fast BF16 is available on GA100/A100-class Ampere, Ada, Hopper, + # and newer architectures. GA10x Ampere cards such as RTX 30xx expose + # BF16 in some software paths but are slow for this workload. + if major >= 9: + return True + if (major, minor) >= (8, 9): + return True + if (major, minor) == (8, 0) and ("a100" in name or "a800" in name): + return True + return False + except Exception: + return False + def get_orig_shape(reader, tensor_name): field_key = f"comfy.gguf.orig_shape.{tensor_name}" field = reader.get_field(field_key) @@ -116,6 +146,7 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F # main loading loop state_dict = {} qtype_dict = {} + bf16_storage_dtype = torch.bfloat16 if device_supports_bf16() else torch.float16 for sd_key, tensor in tensors: tensor_name = tensor.name # torch_tensor = torch.from_numpy(tensor.data) # mmap @@ -135,20 +166,17 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F shape = shape[:-1] # add to state dict - if dynamic: - if tensor.tensor_type not in {gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16}: - state_dict[sd_key] = make_quantized(torch_tensor, tensor.tensor_type, shape) - else: - state_dict[sd_key] = torch_tensor.view(*shape) + if tensor.tensor_type == gguf.GGMLQuantizationType.BF16: + # dtype = torch.float32 if torch_tensor.numel() // 2 <= 10000 else bf16_storage_dtype + torch_tensor = torch_tensor.reshape(-1) + state_dict[sd_key] = torch_tensor.view(torch.bfloat16).reshape(*shape).to(bf16_storage_dtype) + elif tensor.tensor_type in {gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16}: + state_dict[sd_key] = torch_tensor.view(*shape) + elif dynamic: + state_dict[sd_key] = make_quantized(torch_tensor, tensor.tensor_type, shape) else: - if tensor.tensor_type in {gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16}: - torch_tensor = torch_tensor.view(*shape) state_dict[sd_key] = GGMLTensor(torch_tensor, tensor_type=tensor.tensor_type, tensor_shape=shape) - # 1D tensors shouldn't be quantized, this is a fix for BF16 - if len(shape) <= 1 and tensor.tensor_type == gguf.GGMLQuantizationType.BF16: - state_dict[sd_key] = dequantize_tensor(state_dict[sd_key], dtype=torch.float32) - # keep track of loaded tensor types tensor_type_str = getattr(tensor.tensor_type, "name", repr(tensor.tensor_type)) qtype_dict[tensor_type_str] = qtype_dict.get(tensor_type_str, 0) + 1 From a963418d05eb6f443c967d16acebc167bb775d90 Mon Sep 17 00:00:00 2001 From: m8rr Date: Sat, 20 Jun 2026 23:12:21 +0900 Subject: [PATCH 17/34] Add workaround for Qwen3-VL text encoder --- loader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/loader.py b/loader.py index da4d7fc..ced92a9 100644 --- a/loader.py +++ b/loader.py @@ -536,6 +536,9 @@ def gguf_clip_loader(path, dynamic=False): if arch == "qwen2vl": vsd = gguf_mmproj_loader(path) sd.update(vsd) + if arch == "qwen3vl": + sd["model.visual.deepstack_merger_list.0.norm.weight"] = torch.zeros(4608) + sd["model.visual.merger.linear_fc2.weight"] = torch.zeros(4096) else: pass return sd From 249624c1d3250cd8654261c148976bb75636b245 Mon Sep 17 00:00:00 2001 From: m8rr Date: Sun, 21 Jun 2026 03:55:44 +0900 Subject: [PATCH 18/34] Added Vision Support in Qwen3-VL --- loader.py | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/loader.py b/loader.py index ced92a9..9a04a01 100644 --- a/loader.py +++ b/loader.py @@ -254,6 +254,30 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F "ln2.": "norm2.", } +CLIP_VISION_QWEN3_MAP = { + "v.blk": "model.visual.blocks", + "attn_out": "attn.proj", + "ln1": "norm1", + "ln2": "norm2", + "attn_qkv": "attn.qkv", + "ffn_up": "mlp.linear_fc1", + "ffn_down": "mlp.linear_fc2", + "mm.0": "model.visual.merger.linear_fc1", + "mm.2": "model.visual.merger.linear_fc2", + "v.post_ln": "model.visual.merger.norm", + "v.patch_embd": "model.visual.patch_embed.proj", + "v.position_embd.weight": "visual.pos_embed.weight", + "v.deepstack.8.norm": "model.visual.deepstack_merger_list.0.norm", + "v.deepstack.8.fc1": "model.visual.deepstack_merger_list.0.linear_fc1", + "v.deepstack.8.fc2": "model.visual.deepstack_merger_list.0.linear_fc2", + "v.deepstack.16.norm": "model.visual.deepstack_merger_list.1.norm", + "v.deepstack.16.fc1": "model.visual.deepstack_merger_list.1.linear_fc1", + "v.deepstack.16.fc2": "model.visual.deepstack_merger_list.1.linear_fc2", + "v.deepstack.24.norm": "model.visual.deepstack_merger_list.2.norm", + "v.deepstack.24.fc1": "model.visual.deepstack_merger_list.2.linear_fc1", + "v.deepstack.24.fc2": "model.visual.deepstack_merger_list.2.linear_fc2", +} + def sd_map_replace(raw_sd, key_map): sd = {} for k,v in raw_sd.items(): @@ -340,7 +364,15 @@ def gguf_mmproj_loader(path): w2 = dequantize_tensor(vsd.pop("v.patch_embd.weight.1"), dtype=torch.float32) vsd["v.patch_embd.weight"] = torch.stack([w1, w2], dim=2) - # run main replacement + + # qwen3vl + if "v.deepstack.8.norm.weight" in vsd: + for k in list(vsd.keys()): + vsd[k] = dequantize_tensor(vsd[k], dtype=torch.float32) + return sd_map_replace(vsd, CLIP_VISION_QWEN3_MAP) + + + # qwen2vl vsd = sd_map_replace(vsd, CLIP_VISION_SD_MAP) # handle split Q/K/V @@ -533,12 +565,9 @@ def gguf_clip_loader(path, dynamic=False): sd = sd_map_replace(sd, LLAMA_SD_MAP) if arch == "llama": sd = llama_permute(sd, 32, 8) # L3 / Mistral - if arch == "qwen2vl": + if arch == "qwen2vl" or arch == "qwen3vl": vsd = gguf_mmproj_loader(path) sd.update(vsd) - if arch == "qwen3vl": - sd["model.visual.deepstack_merger_list.0.norm.weight"] = torch.zeros(4608) - sd["model.visual.merger.linear_fc2.weight"] = torch.zeros(4096) else: pass return sd From 6a495b21c4ce5d3756bc8825071ad8872fcbd235 Mon Sep 17 00:00:00 2001 From: m8rr Date: Sun, 21 Jun 2026 05:03:04 +0900 Subject: [PATCH 19/34] Allow Qwen3-VL to run without mmproj --- loader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/loader.py b/loader.py index 9a04a01..730a40f 100644 --- a/loader.py +++ b/loader.py @@ -349,7 +349,7 @@ def gguf_mmproj_loader(path): target.append(fname) if len(target) == 0: - logging.error(f"Error: Can't find mmproj file for '{tenc_fname}' (matching:'{tenc}')! Qwen-Image-Edit will be broken!") + logging.error(f"Error: Can't find mmproj file for '{tenc_fname}' (matching:'{tenc}')!") return {} if len(target) > 1: logging.error(f"Ambiguous mmproj for text encoder '{tenc_fname}', will use first match.") @@ -567,7 +567,11 @@ def gguf_clip_loader(path, dynamic=False): sd = llama_permute(sd, 32, 8) # L3 / Mistral if arch == "qwen2vl" or arch == "qwen3vl": vsd = gguf_mmproj_loader(path) - sd.update(vsd) + if not vsd and arch == "qwen3vl": + sd["model.visual.deepstack_merger_list.0.norm.weight"] = torch.zeros(4608) + sd["model.visual.merger.linear_fc2.weight"] = torch.zeros(4096) + else: + sd.update(vsd) else: pass return sd From b7d7413c63e552746ac98ed7fdfdae2a719ee804 Mon Sep 17 00:00:00 2001 From: m8rr Date: Mon, 22 Jun 2026 12:20:45 +0900 Subject: [PATCH 20/34] fix: add Dynamic-VRAM to mmproj loader --- loader.py | 43 ++++--------------------------------------- 1 file changed, 4 insertions(+), 39 deletions(-) diff --git a/loader.py b/loader.py index 730a40f..a340da6 100644 --- a/loader.py +++ b/loader.py @@ -14,36 +14,6 @@ TXT_ARCH_LIST = {"t5", "t5encoder", "llama", "qwen2vl", "qwen3", "qwen3vl", "gemma3"} VIS_TYPE_LIST = {"clip-vision", "mmproj"} -def device_supports_bf16(): - """ - Return True if the active torch device can run BF16 efficiently. Some - consumer Ampere GPUs report BF16 support through PyTorch/Comfy but route - model execution through FP32 manual casts, which is much slower. Keep this - stricter than Comfy's generic helper for GGUF tensor loading. - """ - try: - import comfy.model_management - device = torch.device(comfy.model_management.get_torch_device()) - if device.type != "cuda": - return comfy.model_management.should_use_bf16(device) - - index = device.index if device.index is not None else torch.cuda.current_device() - major, minor = torch.cuda.get_device_capability(index) - name = torch.cuda.get_device_name(index).lower() - - # Native fast BF16 is available on GA100/A100-class Ampere, Ada, Hopper, - # and newer architectures. GA10x Ampere cards such as RTX 30xx expose - # BF16 in some software paths but are slow for this workload. - if major >= 9: - return True - if (major, minor) >= (8, 9): - return True - if (major, minor) == (8, 0) and ("a100" in name or "a800" in name): - return True - return False - except Exception: - return False - def get_orig_shape(reader, tensor_name): field_key = f"comfy.gguf.orig_shape.{tensor_name}" field = reader.get_field(field_key) @@ -146,7 +116,6 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F # main loading loop state_dict = {} qtype_dict = {} - bf16_storage_dtype = torch.bfloat16 if device_supports_bf16() else torch.float16 for sd_key, tensor in tensors: tensor_name = tensor.name # torch_tensor = torch.from_numpy(tensor.data) # mmap @@ -167,9 +136,7 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F # add to state dict if tensor.tensor_type == gguf.GGMLQuantizationType.BF16: - # dtype = torch.float32 if torch_tensor.numel() // 2 <= 10000 else bf16_storage_dtype - torch_tensor = torch_tensor.reshape(-1) - state_dict[sd_key] = torch_tensor.view(torch.bfloat16).reshape(*shape).to(bf16_storage_dtype) + state_dict[sd_key] = torch_tensor.view(torch.bfloat16).reshape(*shape) elif tensor.tensor_type in {gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16}: state_dict[sd_key] = torch_tensor.view(*shape) elif dynamic: @@ -327,7 +294,7 @@ def strip_quant_suffix(name): name = name[:match.start()] return name -def gguf_mmproj_loader(path): +def gguf_mmproj_loader(path, dynamic=False): # Reverse version of Qwen2VLVisionModel.modify_tensors logging.info("Attenpting to find mmproj file for text encoder...") @@ -356,7 +323,7 @@ def gguf_mmproj_loader(path): logging.info(f"Using mmproj '{target[0]}' for text encoder '{tenc_fname}'.") target = os.path.join(root, target[0]) - vsd, _ = gguf_sd_loader(target, is_text_model=True) + vsd, _ = gguf_sd_loader(target, is_text_model=True, dynamic=dynamic) # concat 4D to 5D if "v.patch_embd.weight.1" in vsd: @@ -367,8 +334,6 @@ def gguf_mmproj_loader(path): # qwen3vl if "v.deepstack.8.norm.weight" in vsd: - for k in list(vsd.keys()): - vsd[k] = dequantize_tensor(vsd[k], dtype=torch.float32) return sd_map_replace(vsd, CLIP_VISION_QWEN3_MAP) @@ -566,7 +531,7 @@ def gguf_clip_loader(path, dynamic=False): if arch == "llama": sd = llama_permute(sd, 32, 8) # L3 / Mistral if arch == "qwen2vl" or arch == "qwen3vl": - vsd = gguf_mmproj_loader(path) + vsd = gguf_mmproj_loader(path, dynamic=dynamic) if not vsd and arch == "qwen3vl": sd["model.visual.deepstack_merger_list.0.norm.weight"] = torch.zeros(4608) sd["model.visual.merger.linear_fc2.weight"] = torch.zeros(4096) From 00edba04dde0e694d65e181cf1c874f7cf7389b1 Mon Sep 17 00:00:00 2001 From: m8rr Date: Mon, 22 Jun 2026 15:57:26 +0900 Subject: [PATCH 21/34] Draft: Gemma 4 vision support, DVRAM+BF16 --- loader.py | 97 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 88 insertions(+), 9 deletions(-) diff --git a/loader.py b/loader.py index a340da6..49d4cb7 100644 --- a/loader.py +++ b/loader.py @@ -11,7 +11,7 @@ from .quant_ops import make_quantized IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "hidream", "cosmos", "ltxv", "hyvid", "wan", "lumina2", "qwen_image", "ideogram4"} -TXT_ARCH_LIST = {"t5", "t5encoder", "llama", "qwen2vl", "qwen3", "qwen3vl", "gemma3"} +TXT_ARCH_LIST = {"t5", "t5encoder", "llama", "qwen2vl", "qwen3", "qwen3vl", "gemma3", "gemma4"} VIS_TYPE_LIST = {"clip-vision", "mmproj"} def get_orig_shape(reader, tensor_name): @@ -139,7 +139,7 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F state_dict[sd_key] = torch_tensor.view(torch.bfloat16).reshape(*shape) elif tensor.tensor_type in {gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16}: state_dict[sd_key] = torch_tensor.view(*shape) - elif dynamic: + elif dynamic and sd_key != "per_layer_token_embd.weight": state_dict[sd_key] = make_quantized(torch_tensor, tensor.tensor_type, shape) else: state_dict[sd_key] = GGMLTensor(torch_tensor, tensor_type=tensor.tensor_type, tensor_shape=shape) @@ -208,6 +208,19 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F "post_attention_norm": "post_attention_layernorm", }) +GEMMA4_SD_MAP = { + "per_layer_token_embd": "model.embed_tokens_per_layer", + "per_layer_model_proj": "model.per_layer_model_projection", + "proj.weight": "per_layer_projection.weight", +} +GEMMA4_SD_MAP.update(GEMMA3_SD_MAP) +GEMMA4_SD_MAP.update({ + "layer_output_scale.weight": "layer_scalar", + "inp_gate.weight": "per_layer_input_gate.weight", + "post_norm.weight": "post_per_layer_input_norm.weight", + "per_layer_proj_norm": "model.per_layer_projection_norm", +}) + CLIP_VISION_SD_MAP = { "mm.": "visual.merger.mlp.", "v.post_ln.": "visual.merger.ln_q.", @@ -245,6 +258,30 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F "v.deepstack.24.fc2": "model.visual.deepstack_merger_list.2.linear_fc2", } +CLIP_VISION_GEMMA4_MAP = { + "v.position_embd.weight": "vision_model.patch_embedder.position_embedding_table", + "mm.input_projection": "multi_modal_projector.embedding_projection", + "mm.a.input_projection": "audio_projector.embedding_projection", + "attn_post_norm": "post_attention_layernorm", + "ffn_post_norm": "post_feedforward_layernorm", + "attn_k_norm": "self_attn.k_norm", + "attn_q_norm": "self_attn.q_norm", + "ln1.weight": "input_layernorm.weight", + "ln2.weight": "pre_feedforward_layernorm.weight", + "attn_out.": "self_attn.o_proj.", + "attn_k.": "self_attn.k_proj.", + "attn_q.": "self_attn.q_proj.", + "attn_v.": "self_attn.v_proj.", + "ffn_down.": "mlp.down_proj.", + "ffn_gate.": "mlp.gate_proj.", + "ffn_up.": "mlp.up_proj.", + "r0.weight": "r0.conv.weight", + "r1.weight": "r1.conv.weight", + "v.blk": "vision_model.encoder.layers", + "_proj.weight": "_proj.linear.weight", + "v.patch_embd.": "vision_model.patch_embedder.input_proj.", +} + def sd_map_replace(raw_sd, key_map): sd = {} for k,v in raw_sd.items(): @@ -325,18 +362,21 @@ def gguf_mmproj_loader(path, dynamic=False): target = os.path.join(root, target[0]) vsd, _ = gguf_sd_loader(target, is_text_model=True, dynamic=dynamic) + # gemma4 + if "mm.a.input_projection.weight" in vsd: + vsd["v.patch_embd.weight"] = vsd["v.patch_embd.weight"].permute(0, 2, 3, 1).flatten(start_dim=1) + return sd_map_replace(vsd, CLIP_VISION_GEMMA4_MAP) + # concat 4D to 5D if "v.patch_embd.weight.1" in vsd: w1 = dequantize_tensor(vsd.pop("v.patch_embd.weight"), dtype=torch.float32) w2 = dequantize_tensor(vsd.pop("v.patch_embd.weight.1"), dtype=torch.float32) vsd["v.patch_embd.weight"] = torch.stack([w1, w2], dim=2) - # qwen3vl if "v.deepstack.8.norm.weight" in vsd: return sd_map_replace(vsd, CLIP_VISION_QWEN3_MAP) - # qwen2vl vsd = sd_map_replace(vsd, CLIP_VISION_SD_MAP) @@ -499,6 +539,35 @@ def gguf_gemma3_tokenizer_loader(path): del reader return torch.ByteTensor(list(spm.SerializeToString())) +def gguf_json_tokenizer_loader(path): + tenc_fname = os.path.basename(path) + tenc = os.path.splitext(tenc_fname)[0].lower() + tenc = strip_quant_suffix(tenc) + + target = [] + root = os.path.dirname(path) + for fname in os.listdir(root): + name, ext = os.path.splitext(fname) + if ext.lower() != ".json": + continue + if "tokenizer" not in name.lower(): + continue + if tenc in name.lower(): + target.append(fname) + + if len(target) == 0: + logging.error(f"Error: Can't find tokenizer file for '{tenc_fname}' (matching:'{tenc}')!") + return torch.tensor([], dtype=torch.uint8) + if len(target) > 1: + logging.error(f"Ambiguous tokenizer for text encoder '{tenc_fname}', will use first match.") + + logging.info(f"Using tokenizer '{target[0]}' for text encoder '{tenc_fname}'.") + target = os.path.join(root, target[0]) + + with open(target, "rb") as f: + tokenizer_bytes = f.read() + return torch.tensor(list(tokenizer_bytes), dtype=torch.uint8) + def gguf_clip_loader(path, dynamic=False): sd, extra = gguf_sd_loader(path, is_text_model=True, dynamic=dynamic) arch = extra.get("arch_str", None) @@ -511,7 +580,7 @@ def gguf_clip_loader(path, dynamic=False): logging.warning(f"Dequantizing {temb_key} to prevent runtime OOM.") sd[temb_key] = dequantize_tensor(sd[temb_key], dtype=torch.float16) sd = sd_map_replace(sd, T5_SD_MAP) - elif arch in {"llama", "qwen2vl", "qwen3", "qwen3vl", "gemma3"}: + elif arch in {"llama", "qwen2vl", "qwen3", "qwen3vl", "gemma3", "gemma4"}: # TODO: pass model_options["vocab_size"] to loader somehow temb_key = "token_embd.weight" if temb_key in sd and sd[temb_key].shape[0] >= (64 * 1024): @@ -520,17 +589,27 @@ def gguf_clip_loader(path, dynamic=False): sd["tekken_model"] = gguf_tekken_tokenizer_loader(path, sd[temb_key].shape) elif arch == "gemma3": sd["spiece_model"] = gguf_gemma3_tokenizer_loader(path) - # See note above for T5. - logging.warning(f"Dequantizing {temb_key} to prevent runtime OOM.") - sd[temb_key] = dequantize_tensor(sd[temb_key], dtype=torch.float16) + if arch == "gemma4": + sd["tokenizer_json"] = gguf_json_tokenizer_loader(path) + else: + # See note above for T5. + logging.warning(f"Dequantizing {temb_key} to prevent runtime OOM.") + sd[temb_key] = dequantize_tensor(sd[temb_key], dtype=torch.float16) if arch == "gemma3": sd = sd_map_replace(sd, GEMMA3_SD_MAP) sd = gemma3_norm_corrections(sd) + elif arch == "gemma4": + sd = sd_map_replace(sd, GEMMA4_SD_MAP) + + # temporary workaround + for k in list(sd.keys()): + if k != "tokenizer_json": + sd[k] = dequantize_tensor(sd[k], dtype=torch.bfloat16) else: sd = sd_map_replace(sd, LLAMA_SD_MAP) if arch == "llama": sd = llama_permute(sd, 32, 8) # L3 / Mistral - if arch == "qwen2vl" or arch == "qwen3vl": + if arch == "qwen2vl" or arch == "qwen3vl" or arch == "gemma4": vsd = gguf_mmproj_loader(path, dynamic=dynamic) if not vsd and arch == "qwen3vl": sd["model.visual.deepstack_merger_list.0.norm.weight"] = torch.zeros(4608) From 949a9cb4991866b360925609b497545058b75b4a Mon Sep 17 00:00:00 2001 From: m8rr Date: Mon, 22 Jun 2026 20:45:22 +0900 Subject: [PATCH 22/34] Refactor: Gemma4 now works with --disable-dynamic-vram --- loader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/loader.py b/loader.py index 49d4cb7..065ff6a 100644 --- a/loader.py +++ b/loader.py @@ -139,7 +139,7 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F state_dict[sd_key] = torch_tensor.view(torch.bfloat16).reshape(*shape) elif tensor.tensor_type in {gguf.GGMLQuantizationType.F32, gguf.GGMLQuantizationType.F16}: state_dict[sd_key] = torch_tensor.view(*shape) - elif dynamic and sd_key != "per_layer_token_embd.weight": + elif dynamic: state_dict[sd_key] = make_quantized(torch_tensor, tensor.tensor_type, shape) else: state_dict[sd_key] = GGMLTensor(torch_tensor, tensor_type=tensor.tensor_type, tensor_shape=shape) @@ -602,9 +602,9 @@ def gguf_clip_loader(path, dynamic=False): sd = sd_map_replace(sd, GEMMA4_SD_MAP) # temporary workaround - for k in list(sd.keys()): - if k != "tokenizer_json": - sd[k] = dequantize_tensor(sd[k], dtype=torch.bfloat16) + sd["model.embed_tokens.weight"] = dequantize_tensor(sd["model.embed_tokens.weight"], dtype=torch.bfloat16) + sd["model.embed_tokens_per_layer.weight"] = dequantize_tensor(sd["model.embed_tokens_per_layer.weight"], dtype=torch.bfloat16).as_subclass(torch.Tensor) + sd["model.norm.weight"] = dequantize_tensor(sd["model.norm.weight"], dtype=torch.bfloat16) else: sd = sd_map_replace(sd, LLAMA_SD_MAP) if arch == "llama": From 5a6b74229b3ddca756bea5f74332a14fc76693bd Mon Sep 17 00:00:00 2001 From: m8rr Date: Tue, 23 Jun 2026 09:45:43 +0900 Subject: [PATCH 23/34] Generate tokenizer from Gemma4 GGUF metadata --- loader.py | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 4 deletions(-) diff --git a/loader.py b/loader.py index 065ff6a..2fa165f 100644 --- a/loader.py +++ b/loader.py @@ -5,6 +5,7 @@ import gguf import re import os +import json from .ops import GGMLTensor from .dequant import is_quantized, dequantize_tensor @@ -539,6 +540,88 @@ def gguf_gemma3_tokenizer_loader(path): del reader return torch.ByteTensor(list(spm.SerializeToString())) +def gguf_gemma4_tokenizer_loader(path): + # convert gguf tokenizer to spiece + logging.info("Attempting to recreate tokenizer from GGUF file metadata...") + + reader = gguf.GGUFReader(path) + tokens = get_list_field(reader, "tokenizer.ggml.tokens", str) + merges = get_list_field(reader, "tokenizer.ggml.merges", str) + del reader + + if not tokens or not merges: + raise ValueError("Missing tokenizer metadata") + + vocab = {token: idx for idx, token in enumerate(tokens)} + target_special_ids = [ + 0, 1, 2, 3, 4, 46, 47, 48, 49, 50, 51, 52, 98, 100, 101, 105, 106, 255999, 256000, 258880, 258881, 258882, 258883, 258884 + ] + + added_tokens = [] + for sp_id in target_special_ids: + if sp_id < len(tokens): + added_tokens.append({ + "id": sp_id, + "content": tokens[sp_id], + "single_word": False, + "lstrip": False, + "rstrip": False, + "normalized": False, + "special": True + }) + + tokenizer_dict = { + "version": "1.0", + "truncation": None, + "padding": None, + "added_tokens": added_tokens, + "normalizer": { + "type": "Replace", + "pattern": {"String": " "}, + "content": "\u2581" + }, + "pre_tokenizer": { + "type": "Split", + "pattern": {"String": " "}, + "behavior": "MergedWithPrevious", + "invert": False + }, + "post_processor": { + "type": "TemplateProcessing", + "single": [{"Sequence": {"id": "A", "type_id": 0}}], + "pair": [ + {"Sequence": {"id": "A", "type_id": 0}}, + {"Sequence": {"id": "B", "type_id": 1}} + ], + "special_tokens": {} + }, + "decoder": { + "type": "Sequence", + "decoders": [ + {"type": "Replace", "pattern": {"String": "\u2581"}, "content": " "}, + {"type": "ByteFallback"}, + {"type": "Fuse"} + ] + }, + "model": { + "type": "BPE", + "dropout": None, + "unk_token": "", + "continuing_subword_prefix": None, + "end_of_word_suffix": None, + "fuse_unk": True, + "byte_fallback": True, + "ignore_merges": False, + "vocab": vocab, + "merges": merges + } + } + + json_string = json.dumps(tokenizer_dict, ensure_ascii=False) + + logging.info(f"Created tokenizer with vocab size of {len(vocab)}") + return torch.frombuffer(bytearray(json_string.encode('utf-8')), dtype=torch.uint8) + def gguf_json_tokenizer_loader(path): tenc_fname = os.path.basename(path) tenc = os.path.splitext(tenc_fname)[0].lower() @@ -556,17 +639,17 @@ def gguf_json_tokenizer_loader(path): target.append(fname) if len(target) == 0: - logging.error(f"Error: Can't find tokenizer file for '{tenc_fname}' (matching:'{tenc}')!") - return torch.tensor([], dtype=torch.uint8) + logging.info(f"Can't find tokenizer file for '{tenc_fname}' (matching:'{tenc}')!") + return None if len(target) > 1: - logging.error(f"Ambiguous tokenizer for text encoder '{tenc_fname}', will use first match.") + logging.info(f"Ambiguous tokenizer for text encoder '{tenc_fname}', will use first match.") logging.info(f"Using tokenizer '{target[0]}' for text encoder '{tenc_fname}'.") target = os.path.join(root, target[0]) with open(target, "rb") as f: tokenizer_bytes = f.read() - return torch.tensor(list(tokenizer_bytes), dtype=torch.uint8) + return torch.frombuffer(bytearray(tokenizer_bytes), dtype=torch.uint8) def gguf_clip_loader(path, dynamic=False): sd, extra = gguf_sd_loader(path, is_text_model=True, dynamic=dynamic) @@ -591,6 +674,8 @@ def gguf_clip_loader(path, dynamic=False): sd["spiece_model"] = gguf_gemma3_tokenizer_loader(path) if arch == "gemma4": sd["tokenizer_json"] = gguf_json_tokenizer_loader(path) + if sd["tokenizer_json"] is None: + sd["tokenizer_json"] = gguf_gemma4_tokenizer_loader(path) else: # See note above for T5. logging.warning(f"Dequantizing {temb_key} to prevent runtime OOM.") From c7fb70331a5404c48d573bd052e0ea1b5c4b7675 Mon Sep 17 00:00:00 2001 From: m8rr Date: Tue, 23 Jun 2026 12:05:21 +0900 Subject: [PATCH 24/34] Qwen3VL-4B support --- loader.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/loader.py b/loader.py index 2fa165f..2b030d5 100644 --- a/loader.py +++ b/loader.py @@ -237,6 +237,13 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F CLIP_VISION_QWEN3_MAP = { "v.blk": "model.visual.blocks", + ".fc": ".linear_fc", + "ck.8.": "st.0.", + "ck.16.": "st.1.", + "ck.24.": "st.2.", + "ck.5.": "st.0.", + "ck.11.": "st.1.", + "ck.17.": "st.2.", "attn_out": "attn.proj", "ln1": "norm1", "ln2": "norm2", @@ -248,15 +255,7 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F "v.post_ln": "model.visual.merger.norm", "v.patch_embd": "model.visual.patch_embed.proj", "v.position_embd.weight": "visual.pos_embed.weight", - "v.deepstack.8.norm": "model.visual.deepstack_merger_list.0.norm", - "v.deepstack.8.fc1": "model.visual.deepstack_merger_list.0.linear_fc1", - "v.deepstack.8.fc2": "model.visual.deepstack_merger_list.0.linear_fc2", - "v.deepstack.16.norm": "model.visual.deepstack_merger_list.1.norm", - "v.deepstack.16.fc1": "model.visual.deepstack_merger_list.1.linear_fc1", - "v.deepstack.16.fc2": "model.visual.deepstack_merger_list.1.linear_fc2", - "v.deepstack.24.norm": "model.visual.deepstack_merger_list.2.norm", - "v.deepstack.24.fc1": "model.visual.deepstack_merger_list.2.linear_fc1", - "v.deepstack.24.fc2": "model.visual.deepstack_merger_list.2.linear_fc2", + "v.deepstast.": "model.visual.deepstack_merger_list.", } CLIP_VISION_GEMMA4_MAP = { @@ -375,7 +374,7 @@ def gguf_mmproj_loader(path, dynamic=False): vsd["v.patch_embd.weight"] = torch.stack([w1, w2], dim=2) # qwen3vl - if "v.deepstack.8.norm.weight" in vsd: + if any("deepstack" in key for key in vsd): return sd_map_replace(vsd, CLIP_VISION_QWEN3_MAP) # qwen2vl @@ -697,8 +696,9 @@ def gguf_clip_loader(path, dynamic=False): if arch == "qwen2vl" or arch == "qwen3vl" or arch == "gemma4": vsd = gguf_mmproj_loader(path, dynamic=dynamic) if not vsd and arch == "qwen3vl": - sd["model.visual.deepstack_merger_list.0.norm.weight"] = torch.zeros(4608) - sd["model.visual.merger.linear_fc2.weight"] = torch.zeros(4096) + weight = sd["model.norm.weight"].shape[0] + sd["model.visual.deepstack_merger_list.0.norm.weight"] = torch.zeros(4608 if weight == 4096 else 4096) + sd["model.visual.merger.linear_fc2.weight"] = torch.zeros(weight) else: sd.update(vsd) else: From a1641cb195179593c130a274f831150ac149cf86 Mon Sep 17 00:00:00 2001 From: m8rr Date: Tue, 23 Jun 2026 13:13:31 +0900 Subject: [PATCH 25/34] refactor: change tokenizer return --- loader.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/loader.py b/loader.py index 2b030d5..b921e64 100644 --- a/loader.py +++ b/loader.py @@ -5,7 +5,6 @@ import gguf import re import os -import json from .ops import GGMLTensor from .dequant import is_quantized, dequantize_tensor @@ -452,7 +451,7 @@ def gguf_tokenizer_loader(path, temb_shape): logging.info(f"Created tokenizer with vocab size of {len(spm.pieces)}") del reader - return torch.ByteTensor(list(spm.SerializeToString())) + return torch.frombuffer(bytearray(spm.SerializeToString()), dtype=torch.uint8) def gguf_tekken_tokenizer_loader(path, temb_shape): # convert ggml (hf) tokenizer metadata to tekken/comfy data @@ -495,7 +494,7 @@ def gguf_tekken_tokenizer_loader(path, temb_shape): logging.info(f"Created tekken tokenizer with vocab size of {len(data['vocab'])} (+{len(data['special_tokens'])})") del reader - return torch.ByteTensor(list(json.dumps(data).encode('utf-8'))) + return torch.frombuffer(bytearray(json.dumps(data).encode('utf-8')), dtype=torch.uint8) def gguf_gemma3_tokenizer_loader(path): #TODO: merge into gguf_tokenizer_loader @@ -537,11 +536,12 @@ def gguf_gemma3_tokenizer_loader(path): logging.info(f"Created tokenizer with vocab size of {len(spm.pieces)}") del reader - return torch.ByteTensor(list(spm.SerializeToString())) + return torch.frombuffer(bytearray(spm.SerializeToString()), dtype=torch.uint8) def gguf_gemma4_tokenizer_loader(path): # convert gguf tokenizer to spiece logging.info("Attempting to recreate tokenizer from GGUF file metadata...") + import json reader = gguf.GGUFReader(path) tokens = get_list_field(reader, "tokenizer.ggml.tokens", str) From 71875559d542b6817263ecb9fa740967314575ad Mon Sep 17 00:00:00 2001 From: m8rr Date: Tue, 23 Jun 2026 17:20:03 +0900 Subject: [PATCH 26/34] Reuse GGUFReader for tokenizer loader --- loader.py | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/loader.py b/loader.py index b921e64..50c3ed4 100644 --- a/loader.py +++ b/loader.py @@ -162,6 +162,8 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F "arch_str": arch_str, "metadata": get_gguf_metadata(reader) } + if is_text_model: + extra["reader"] = reader return (state_dict, extra) # for remapping llama.cpp -> original key names @@ -405,7 +407,7 @@ def gguf_mmproj_loader(path, dynamic=False): return vsd -def gguf_tokenizer_loader(path, temb_shape): +def gguf_tokenizer_loader(path, temb_shape, reader): # convert gguf tokenizer to spiece logging.info("Attempting to recreate sentencepiece tokenizer from GGUF file metadata...") try: @@ -414,8 +416,6 @@ def gguf_tokenizer_loader(path, temb_shape): raise ImportError("Please make sure sentencepiece and protobuf are installed.\npip install sentencepiece protobuf") spm = model.ModelProto() - reader = gguf.GGUFReader(path) - if get_field(reader, "tokenizer.ggml.model", str) == "t5": if temb_shape == (256384, 4096): # probably UMT5 spm.trainer_spec.model_type = 1 # Unigram (do we have a T5 w/ BPE?) @@ -453,15 +453,13 @@ def gguf_tokenizer_loader(path, temb_shape): del reader return torch.frombuffer(bytearray(spm.SerializeToString()), dtype=torch.uint8) -def gguf_tekken_tokenizer_loader(path, temb_shape): +def gguf_tekken_tokenizer_loader(path, temb_shape, reader): # convert ggml (hf) tokenizer metadata to tekken/comfy data logging.info("Attempting to recreate tekken tokenizer from GGUF file metadata...") import json import base64 from transformers.convert_slow_tokenizer import bytes_to_unicode - reader = gguf.GGUFReader(path) - model_str = get_field(reader, "tokenizer.ggml.model", str) if model_str == "gpt2": if temb_shape == (131072, 5120): # probably Mistral @@ -496,7 +494,7 @@ def gguf_tekken_tokenizer_loader(path, temb_shape): del reader return torch.frombuffer(bytearray(json.dumps(data).encode('utf-8')), dtype=torch.uint8) -def gguf_gemma3_tokenizer_loader(path): +def gguf_gemma3_tokenizer_loader(path, reader): #TODO: merge into gguf_tokenizer_loader logging.info("Attempting to recreate sentencepiece tokenizer from GGUF file metadata...") try: @@ -504,7 +502,6 @@ def gguf_gemma3_tokenizer_loader(path): except ImportError: raise ImportError("Please install sentencepiece and protobuf.\npip install sentencepiece protobuf") spm = model.ModelProto() - reader = gguf.GGUFReader(path) spm.normalizer_spec.name = "identity" spm.normalizer_spec.add_dummy_prefix = False @@ -538,12 +535,11 @@ def gguf_gemma3_tokenizer_loader(path): del reader return torch.frombuffer(bytearray(spm.SerializeToString()), dtype=torch.uint8) -def gguf_gemma4_tokenizer_loader(path): +def gguf_gemma4_tokenizer_loader(path, reader): # convert gguf tokenizer to spiece logging.info("Attempting to recreate tokenizer from GGUF file metadata...") import json - reader = gguf.GGUFReader(path) tokens = get_list_field(reader, "tokenizer.ggml.tokens", str) merges = get_list_field(reader, "tokenizer.ggml.merges", str) del reader @@ -657,7 +653,7 @@ def gguf_clip_loader(path, dynamic=False): temb_key = "token_embd.weight" if temb_key in sd and sd[temb_key].shape == (256384, 4096): # non-standard Comfy-Org tokenizer - sd["spiece_model"] = gguf_tokenizer_loader(path, sd[temb_key].shape) + sd["spiece_model"] = gguf_tokenizer_loader(path, sd[temb_key].shape, extra.pop("reader")) # TODO: dequantizing token embed here is janky but otherwise we OOM due to tensor being massive. logging.warning(f"Dequantizing {temb_key} to prevent runtime OOM.") sd[temb_key] = dequantize_tensor(sd[temb_key], dtype=torch.float16) @@ -668,13 +664,11 @@ def gguf_clip_loader(path, dynamic=False): if temb_key in sd and sd[temb_key].shape[0] >= (64 * 1024): if arch == "llama" and sd[temb_key].shape == (131072, 5120): # non-standard Comfy-Org tokenizer - sd["tekken_model"] = gguf_tekken_tokenizer_loader(path, sd[temb_key].shape) + sd["tekken_model"] = gguf_tekken_tokenizer_loader(path, sd[temb_key].shape, extra.pop("reader")) elif arch == "gemma3": - sd["spiece_model"] = gguf_gemma3_tokenizer_loader(path) + sd["spiece_model"] = gguf_gemma3_tokenizer_loader(path, extra.pop("reader")) if arch == "gemma4": - sd["tokenizer_json"] = gguf_json_tokenizer_loader(path) - if sd["tokenizer_json"] is None: - sd["tokenizer_json"] = gguf_gemma4_tokenizer_loader(path) + sd["tokenizer_json"] = gguf_json_tokenizer_loader(path) or gguf_gemma4_tokenizer_loader(path, extra.pop("reader")) else: # See note above for T5. logging.warning(f"Dequantizing {temb_key} to prevent runtime OOM.") From 5b2d245dc53f0fe1d607613646e59f5cd4ca639a Mon Sep 17 00:00:00 2001 From: m8rr Date: Wed, 24 Jun 2026 15:23:40 +0900 Subject: [PATCH 27/34] Stole: KREA-2 support (RealRebelAI/ComfyUI-GGUF_KREA-2) --- loader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader.py b/loader.py index 50c3ed4..124bd68 100644 --- a/loader.py +++ b/loader.py @@ -10,7 +10,7 @@ from .dequant import is_quantized, dequantize_tensor from .quant_ops import make_quantized -IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "hidream", "cosmos", "ltxv", "hyvid", "wan", "lumina2", "qwen_image", "ideogram4"} +IMG_ARCH_LIST = {"flux", "sd1", "sdxl", "sd3", "aura", "hidream", "cosmos", "ltxv", "hyvid", "wan", "lumina2", "qwen_image", "ideogram4", "krea2"} TXT_ARCH_LIST = {"t5", "t5encoder", "llama", "qwen2vl", "qwen3", "qwen3vl", "gemma3", "gemma4"} VIS_TYPE_LIST = {"clip-vision", "mmproj"} From 736fc9c5153b068dcdc0f62d42ba9c046fce8b4e Mon Sep 17 00:00:00 2001 From: m8rr Date: Wed, 24 Jun 2026 22:04:44 +0900 Subject: [PATCH 28/34] Improve GGUFReader loading performance --- loader.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/loader.py b/loader.py index 124bd68..3383711 100644 --- a/loader.py +++ b/loader.py @@ -5,7 +5,9 @@ import gguf import re import os +import numpy as np +from gguf import GGUFReader, GGUFValueType from .ops import GGMLTensor from .dequant import is_quantized, dequantize_tensor from .quant_ops import make_quantized @@ -14,6 +16,61 @@ TXT_ARCH_LIST = {"t5", "t5encoder", "llama", "qwen2vl", "qwen3", "qwen3vl", "gemma3", "gemma4"} VIS_TYPE_LIST = {"clip-vision", "mmproj"} +class LazyGGUFReader(GGUFReader): + def _get_field_parts(self, orig_offs: int, raw_type: int): + gtype = GGUFValueType(raw_type) + + if gtype == GGUFValueType.ARRAY: + raw_itype = self._get(orig_offs, np.uint32) + offs = orig_offs + int(raw_itype.nbytes) + alen = self._get(offs, np.uint64) + array_len = alen[0] + + if array_len > 1000: + offs += int(alen.nbytes) + sub_type = raw_itype[0] + types = [gtype, GGUFValueType(sub_type)] + aparts = [raw_itype, alen] + data_idxs = [] + data_view = self.data + is_swapped = (self.byte_order == 'S') + + if sub_type == 8: + for _ in range(array_len): + slen_arr = data_view[offs : offs + 8] + slen = slen_arr.view(dtype=np.uint64)[0] + if is_swapped: + slen = slen.newbyteorder('S') + + str_total_bytes = 8 + int(slen) + sdata_arr = data_view[offs + 8 : offs + str_total_bytes] + + idxs_offs = len(aparts) + aparts.append(slen_arr) + aparts.append(sdata_arr) + data_idxs.append(idxs_offs + 1) + offs += str_total_bytes + + return offs - orig_offs, aparts, data_idxs, types + else: + nptype = self.gguf_scalar_to_np.get(GGUFValueType(sub_type)) + if nptype is not None: + item_size = np.dtype(nptype).itemsize + total_bytes = array_len * item_size + total_data = data_view[offs : offs + total_bytes].view(dtype=nptype) + if is_swapped: + total_data = total_data.newbyteorder('S') + + idxs_offs = len(aparts) + aparts.extend(total_data[i : i + 1] for i in range(array_len)) + data_idxs = list(range(idxs_offs, idxs_offs + array_len)) + offs += total_bytes + + return offs - orig_offs, aparts, data_idxs, types + + return super()._get_field_parts(orig_offs, raw_type) + + def get_orig_shape(reader, tensor_name): field_key = f"comfy.gguf.orig_shape.{tensor_name}" field = reader.get_field(field_key) @@ -72,7 +129,7 @@ def gguf_sd_loader(path, handle_prefix="model.diffusion_model.", is_text_model=F """ Read state dict as fake tensors """ - reader = gguf.GGUFReader(path) + reader = LazyGGUFReader(path) # filter and strip prefix has_prefix = False From 9a19c7d1ceb28ad752b9e1b74c7b0b3e23eafe6d Mon Sep 17 00:00:00 2001 From: m8rr Date: Thu, 25 Jun 2026 15:41:21 +0900 Subject: [PATCH 29/34] Refactor: logging message handling, Feat. 19e396a --- loader.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/loader.py b/loader.py index 3383711..64970da 100644 --- a/loader.py +++ b/loader.py @@ -391,7 +391,7 @@ def strip_quant_suffix(name): def gguf_mmproj_loader(path, dynamic=False): # Reverse version of Qwen2VLVisionModel.modify_tensors - logging.info("Attenpting to find mmproj file for text encoder...") + logging.info("Attempting to find mmproj file for text encoder...") # get name to match w/o quant suffix tenc_fname = os.path.basename(path) @@ -411,10 +411,10 @@ def gguf_mmproj_loader(path, dynamic=False): target.append(fname) if len(target) == 0: - logging.error(f"Error: Can't find mmproj file for '{tenc_fname}' (matching:'{tenc}')!") + logging.warning(f"Can't find mmproj file for '{tenc_fname}' (matching:'{tenc}'), vision function will not work!") return {} if len(target) > 1: - logging.error(f"Ambiguous mmproj for text encoder '{tenc_fname}', will use first match.") + logging.info(f"Ambiguous mmproj for text encoder '{tenc_fname}', will use first match.") logging.info(f"Using mmproj '{target[0]}' for text encoder '{tenc_fname}'.") target = os.path.join(root, target[0]) @@ -508,7 +508,9 @@ def gguf_tokenizer_loader(path, temb_shape, reader): logging.info(f"Created tokenizer with vocab size of {len(spm.pieces)}") del reader - return torch.frombuffer(bytearray(spm.SerializeToString()), dtype=torch.uint8) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="The given buffer is not writable") + return torch.frombuffer(spm.SerializeToString(), dtype=torch.uint8) def gguf_tekken_tokenizer_loader(path, temb_shape, reader): # convert ggml (hf) tokenizer metadata to tekken/comfy data @@ -549,7 +551,9 @@ def gguf_tekken_tokenizer_loader(path, temb_shape, reader): logging.info(f"Created tekken tokenizer with vocab size of {len(data['vocab'])} (+{len(data['special_tokens'])})") del reader - return torch.frombuffer(bytearray(json.dumps(data).encode('utf-8')), dtype=torch.uint8) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="The given buffer is not writable") + return torch.frombuffer(json.dumps(data).encode('utf-8'), dtype=torch.uint8) def gguf_gemma3_tokenizer_loader(path, reader): #TODO: merge into gguf_tokenizer_loader @@ -590,7 +594,9 @@ def gguf_gemma3_tokenizer_loader(path, reader): logging.info(f"Created tokenizer with vocab size of {len(spm.pieces)}") del reader - return torch.frombuffer(bytearray(spm.SerializeToString()), dtype=torch.uint8) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="The given buffer is not writable") + return torch.frombuffer(spm.SerializeToString(), dtype=torch.uint8) def gguf_gemma4_tokenizer_loader(path, reader): # convert gguf tokenizer to spiece @@ -672,7 +678,9 @@ def gguf_gemma4_tokenizer_loader(path, reader): json_string = json.dumps(tokenizer_dict, ensure_ascii=False) logging.info(f"Created tokenizer with vocab size of {len(vocab)}") - return torch.frombuffer(bytearray(json_string.encode('utf-8')), dtype=torch.uint8) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="The given buffer is not writable") + return torch.frombuffer(json_string.encode('utf-8'), dtype=torch.uint8) def gguf_json_tokenizer_loader(path): tenc_fname = os.path.basename(path) @@ -701,7 +709,9 @@ def gguf_json_tokenizer_loader(path): with open(target, "rb") as f: tokenizer_bytes = f.read() - return torch.frombuffer(bytearray(tokenizer_bytes), dtype=torch.uint8) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message="The given buffer is not writable") + return torch.frombuffer(tokenizer_bytes, dtype=torch.uint8) def gguf_clip_loader(path, dynamic=False): sd, extra = gguf_sd_loader(path, is_text_model=True, dynamic=dynamic) @@ -725,7 +735,7 @@ def gguf_clip_loader(path, dynamic=False): elif arch == "gemma3": sd["spiece_model"] = gguf_gemma3_tokenizer_loader(path, extra.pop("reader")) if arch == "gemma4": - sd["tokenizer_json"] = gguf_json_tokenizer_loader(path) or gguf_gemma4_tokenizer_loader(path, extra.pop("reader")) + sd["tokenizer_json"] = gguf_gemma4_tokenizer_loader(path, extra.pop("reader")) else: # See note above for T5. logging.warning(f"Dequantizing {temb_key} to prevent runtime OOM.") From 1c113ea4f36362dc934f60c60bf6ab7b9e0e1b61 Mon Sep 17 00:00:00 2001 From: m8rr Date: Thu, 25 Jun 2026 23:55:17 +0900 Subject: [PATCH 30/34] fix: resolve torch.compile detection issues --- quant_ops.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/quant_ops.py b/quant_ops.py index 53e84dd..60cf9bc 100644 --- a/quant_ops.py +++ b/quant_ops.py @@ -28,9 +28,13 @@ def register_layout_class(name, cls): from .dequant import dequantize_functions, TORCH_COMPATIBLE_QTYPES, is_quantized -HAS_COMPILE = hasattr(torch, 'compile') COMPILED_DEQUANT_FUNCTIONS = {} - +if hasattr(torch, 'compile'): + for k, func in dequantize_functions.items(): + COMPILED_DEQUANT_FUNCTIONS[k] = torch.compile(func) +else: + COMPILED_DEQUANT_FUNCTIONS = dequantize_functions.copy() + if _CK_AVAILABLE: @dataclass(frozen=True) class GGMLLayoutParams(BaseLayoutParams): @@ -62,8 +66,12 @@ def dequantize(cls, qdata, params): n_blocks = raw.numel() // type_size blocks = raw.reshape((n_blocks, type_size)) - fn = get_compiled(qtype, dequantize_functions[qtype]) - blocks = fn(blocks, block_size, type_size, None) + fn = COMPILED_DEQUANT_FUNCTIONS[qtype] + try: + blocks = fn(blocks, block_size, type_size, None) + except Exception: + fn = COMPILED_DEQUANT_FUNCTIONS[qtype] = dequantize_functions[qtype] + blocks = fn(blocks, block_size, type_size, None) return blocks.reshape(oshape).to(params.orig_dtype) @classmethod @@ -76,21 +84,6 @@ def state_dict_tensors(cls, qdata, params): register_layout_class("GGMLLayout", GGMLLayout) -def get_compiled(qtype, raw_func): - if qtype in COMPILED_DEQUANT_FUNCTIONS: - return COMPILED_DEQUANT_FUNCTIONS[qtype] - - if HAS_COMPILE: - try: - compiled = torch.compile(raw_func, fullgraph=True) - except Exception: - compiled = raw_func - else: - compiled = raw_func - - COMPILED_DEQUANT_FUNCTIONS[qtype] = compiled - return compiled - def make_quantized(qdata, tensor_type, tensor_shape, orig_dtype=torch.float16): """Construct a GGML QuantizedTensor from raw packed data.""" params = GGMLLayoutParams( From 5e98d9cbca52ee02cf08a78724edab3e12cfbcdf Mon Sep 17 00:00:00 2001 From: m8rr Date: Mon, 29 Jun 2026 17:56:28 +0900 Subject: [PATCH 31/34] gemma4 E4B runs, but unknown what side effects there might be --- quant_ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/quant_ops.py b/quant_ops.py index 60cf9bc..a31c2ac 100644 --- a/quant_ops.py +++ b/quant_ops.py @@ -68,10 +68,10 @@ def dequantize(cls, qdata, params): fn = COMPILED_DEQUANT_FUNCTIONS[qtype] try: - blocks = fn(blocks, block_size, type_size, None) + blocks = fn(blocks, block_size, type_size, params.orig_dtype) except Exception: fn = COMPILED_DEQUANT_FUNCTIONS[qtype] = dequantize_functions[qtype] - blocks = fn(blocks, block_size, type_size, None) + blocks = fn(blocks, block_size, type_size, params.orig_dtype) return blocks.reshape(oshape).to(params.orig_dtype) @classmethod From ccfb08e5538cd164cd116909c5d400ad02d02908 Mon Sep 17 00:00:00 2001 From: m8rr Date: Mon, 29 Jun 2026 21:46:06 +0900 Subject: [PATCH 32/34] Refactor: Remove unused argument --- loader.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/loader.py b/loader.py index 64970da..a19902f 100644 --- a/loader.py +++ b/loader.py @@ -464,7 +464,7 @@ def gguf_mmproj_loader(path, dynamic=False): return vsd -def gguf_tokenizer_loader(path, temb_shape, reader): +def gguf_tokenizer_loader(reader, temb_shape): # convert gguf tokenizer to spiece logging.info("Attempting to recreate sentencepiece tokenizer from GGUF file metadata...") try: @@ -512,7 +512,7 @@ def gguf_tokenizer_loader(path, temb_shape, reader): warnings.filterwarnings("ignore", message="The given buffer is not writable") return torch.frombuffer(spm.SerializeToString(), dtype=torch.uint8) -def gguf_tekken_tokenizer_loader(path, temb_shape, reader): +def gguf_tekken_tokenizer_loader(reader, temb_shape): # convert ggml (hf) tokenizer metadata to tekken/comfy data logging.info("Attempting to recreate tekken tokenizer from GGUF file metadata...") import json @@ -555,7 +555,7 @@ def gguf_tekken_tokenizer_loader(path, temb_shape, reader): warnings.filterwarnings("ignore", message="The given buffer is not writable") return torch.frombuffer(json.dumps(data).encode('utf-8'), dtype=torch.uint8) -def gguf_gemma3_tokenizer_loader(path, reader): +def gguf_gemma3_tokenizer_loader(reader): #TODO: merge into gguf_tokenizer_loader logging.info("Attempting to recreate sentencepiece tokenizer from GGUF file metadata...") try: @@ -598,7 +598,7 @@ def gguf_gemma3_tokenizer_loader(path, reader): warnings.filterwarnings("ignore", message="The given buffer is not writable") return torch.frombuffer(spm.SerializeToString(), dtype=torch.uint8) -def gguf_gemma4_tokenizer_loader(path, reader): +def gguf_gemma4_tokenizer_loader(reader): # convert gguf tokenizer to spiece logging.info("Attempting to recreate tokenizer from GGUF file metadata...") import json @@ -720,7 +720,7 @@ def gguf_clip_loader(path, dynamic=False): temb_key = "token_embd.weight" if temb_key in sd and sd[temb_key].shape == (256384, 4096): # non-standard Comfy-Org tokenizer - sd["spiece_model"] = gguf_tokenizer_loader(path, sd[temb_key].shape, extra.pop("reader")) + sd["spiece_model"] = gguf_tokenizer_loader(extra.pop("reader"), sd[temb_key].shape) # TODO: dequantizing token embed here is janky but otherwise we OOM due to tensor being massive. logging.warning(f"Dequantizing {temb_key} to prevent runtime OOM.") sd[temb_key] = dequantize_tensor(sd[temb_key], dtype=torch.float16) @@ -731,11 +731,11 @@ def gguf_clip_loader(path, dynamic=False): if temb_key in sd and sd[temb_key].shape[0] >= (64 * 1024): if arch == "llama" and sd[temb_key].shape == (131072, 5120): # non-standard Comfy-Org tokenizer - sd["tekken_model"] = gguf_tekken_tokenizer_loader(path, sd[temb_key].shape, extra.pop("reader")) + sd["tekken_model"] = gguf_tekken_tokenizer_loader(extra.pop("reader"), sd[temb_key].shape) elif arch == "gemma3": - sd["spiece_model"] = gguf_gemma3_tokenizer_loader(path, extra.pop("reader")) + sd["spiece_model"] = gguf_gemma3_tokenizer_loader(extra.pop("reader")) if arch == "gemma4": - sd["tokenizer_json"] = gguf_gemma4_tokenizer_loader(path, extra.pop("reader")) + sd["tokenizer_json"] = gguf_gemma4_tokenizer_loader(extra.pop("reader")) else: # See note above for T5. logging.warning(f"Dequantizing {temb_key} to prevent runtime OOM.") From e71c2f67ab94c89d0bff72babdb2a348127ba793 Mon Sep 17 00:00:00 2001 From: m8rr Date: Tue, 4 Aug 2026 10:59:18 +0900 Subject: [PATCH 33/34] Support Qwen3 32B --- loader.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/loader.py b/loader.py index a19902f..496383e 100644 --- a/loader.py +++ b/loader.py @@ -754,14 +754,15 @@ def gguf_clip_loader(path, dynamic=False): sd = sd_map_replace(sd, LLAMA_SD_MAP) if arch == "llama": sd = llama_permute(sd, 32, 8) # L3 / Mistral - if arch == "qwen2vl" or arch == "qwen3vl" or arch == "gemma4": + if arch == "qwen2vl" or (arch == "qwen3vl" and "model.norm.weight" in sd) or arch == "gemma4": vsd = gguf_mmproj_loader(path, dynamic=dynamic) if not vsd and arch == "qwen3vl": weight = sd["model.norm.weight"].shape[0] - sd["model.visual.deepstack_merger_list.0.norm.weight"] = torch.zeros(4608 if weight == 4096 else 4096) + sd["model.visual.deepstack_merger_list.0.norm.weight"] = torch.zeros(4096 if weight < 4096 else 4608) sd["model.visual.merger.linear_fc2.weight"] = torch.zeros(weight) else: sd.update(vsd) else: pass return sd + From 853daf6fe2931ef2ce811bb84bc950026fd7d903 Mon Sep 17 00:00:00 2001 From: Nif <64535498+Nif00@users.noreply.github.com> Date: Wed, 5 Aug 2026 03:12:15 +0300 Subject: [PATCH 34/34] Fix MiniMax H3 Qwen3-VL 32B mmproj loading --- loader.py | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/loader.py b/loader.py index 496383e..582bcde 100644 --- a/loader.py +++ b/loader.py @@ -754,14 +754,41 @@ def gguf_clip_loader(path, dynamic=False): sd = sd_map_replace(sd, LLAMA_SD_MAP) if arch == "llama": sd = llama_permute(sd, 32, 8) # L3 / Mistral - if arch == "qwen2vl" or (arch == "qwen3vl" and "model.norm.weight" in sd) or arch == "gemma4": + if arch in {"qwen2vl", "qwen3vl", "gemma4"}: vsd = gguf_mmproj_loader(path, dynamic=dynamic) - if not vsd and arch == "qwen3vl": - weight = sd["model.norm.weight"].shape[0] - sd["model.visual.deepstack_merger_list.0.norm.weight"] = torch.zeros(4096 if weight < 4096 else 4608) - sd["model.visual.merger.linear_fc2.weight"] = torch.zeros(weight) - else: - sd.update(vsd) + + if vsd: + # MiniMax-H3 uses the truncated Qwen3-VL-32B encoder. + # ComfyUI detects it by: + # visual.deepstack_merger_list... + # model.layers.49... + # + # The generic Qwen3-VL mmproj mapper produces model.visual.*, + # which makes ComfyUI incorrectly instantiate Qwen3-VL-8B. + is_minimax_h3 = ( + arch == "qwen3vl" + and "model.layers.49.self_attn.q_proj.weight" in sd + ) + + if is_minimax_h3: + vsd = { + ( + key.replace("model.visual.", "visual.", 1) + if key.startswith("model.visual.") + else key + ): value + for key, value in vsd.items() + } + + sd.update(vsd) + + elif arch == "qwen3vl" and "model.norm.weight" in sd: + # Generic full-model fallback only. + weight = sd["model.norm.weight"].shape[0] + sd["model.visual.deepstack_merger_list.0.norm.weight"] = torch.zeros( + 4096 if weight < 4096 else 4608 + ) + sd["model.visual.merger.linear_fc2.weight"] = torch.zeros(weight) else: pass return sd