diff --git a/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml b/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml index 2adcf1f60f0..057ed77409f 100644 --- a/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml +++ b/modelopt_recipes/configs/ptq/units/default_disabled_quantizers.yaml @@ -36,6 +36,20 @@ enable: false - quantizer_name: 'output.*' enable: false + # Multimodal vision branch: keep the vision encoder (SigLIP / ViT) and any + # multimodal embedding projection in BF16 by default. Recipes that enable bare + # `*weight_quantizer` / `*input_quantizer` or `*mlp*` wildcards otherwise also + # match the vision tower (`model.vision_tower.*`, `model.visual.*`) and the + # embedding projection (`model.embed_vision.*`); quantizing the vision branch + # crashes export / produces garbage image embeddings on VL models (gemma-4, + # Qwen3.5-VL — NVBugs 6293731, 6293762, 6294017). A recipe that intentionally + # quantizes vision must re-enable these after importing this unit. + - quantizer_name: '*embed_vision*' + enable: false + - quantizer_name: '*vision_tower*' + enable: false + - quantizer_name: '*visual*' + enable: false - parent_class: 'nn.BatchNorm1d' quantizer_name: '*' enable: false