Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -1137,8 +1137,8 @@ def __call__(
if controlnet_config.force_zeros_for_pooled_projection:
# instantx sd3 controlnet used zero pooled projection
controlnet_pooled_projections = torch.zeros_like(pooled_prompt_embeds)
else:
controlnet_pooled_projections = controlnet_pooled_projections or pooled_prompt_embeds
elif controlnet_pooled_projections is None:
controlnet_pooled_projections = pooled_prompt_embeds

if controlnet_config.joint_attention_dim is not None:
controlnet_encoder_hidden_states = prompt_embeds
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1269,8 +1269,6 @@ def __call__(

if controlnet_pooled_projections is None:
controlnet_pooled_projections = torch.zeros_like(pooled_prompt_embeds)
else:
controlnet_pooled_projections = controlnet_pooled_projections or pooled_prompt_embeds

# 4. Prepare timesteps
if XLA_AVAILABLE:
Expand Down
30 changes: 30 additions & 0 deletions tests/pipelines/controlnet_sd3/test_controlnet_sd3.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,36 @@ def test_controlnet_sd35(self):
# fmt: on
self._run_and_check_slice(components, expected_slice)

def test_controlnet_pooled_projections_accepts_tensor(self):
# Regression: when the ControlNet does not force zeroed pooled projections, the pipeline
# resolved this argument with `controlnet_pooled_projections or pooled_prompt_embeds`.
# `or` takes the truthiness of the tensor, which raises "Boolean value of Tensor with more
# than one value is ambiguous" for any real pooled projection. See huggingface/diffusers#9686.
components = self.get_dummy_components()
torch.manual_seed(0)
components["controlnet"] = SD3ControlNetModel(
sample_size=32,
patch_size=1,
in_channels=8,
num_layers=1,
attention_head_dim=8,
num_attention_heads=4,
joint_attention_dim=32,
caption_projection_dim=32,
pooled_projection_dim=64,
out_channels=8,
qk_norm="rms_norm",
force_zeros_for_pooled_projection=False,
)
pipe = self.get_pipeline(**components).to(torch_device, dtype=torch.float32)

inputs = self.get_dummy_inputs()
inputs["controlnet_pooled_projections"] = torch.zeros((1, 64), device=torch_device)

image = pipe(**inputs).images

assert image.shape == (1, *self.output_shape)


class TestStableDiffusion3ControlNetPipelineMemory(StableDiffusion3ControlNetPipelineTesterConfig, MemoryTesterMixin):
"""Memory optimization tests (CPU offload, group offload, layerwise casting) for the SD3 ControlNet pipeline."""
Expand Down
Loading