diff --git a/examples/server/api.md b/examples/server/api.md index cff4e8455..f2f6d32b0 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -155,6 +155,10 @@ image establishes the generation dimensions if `size` is omitted. Input geometry follows `image_preprocess`: references preserve their dimensions by default, while init and mask use the generation canvas preset. +When the request starts without preconfigured reference images, the first uploaded image becomes the img2img init image if the loaded model does not support reference-image conditioning, the final reference list is empty, a mask is present, or an explicit numeric `strength` override is provided. Otherwise, only the final reference list is used for image conditioning, without initializing the img2img latent. Requests with preconfigured references retain their existing init-image selection. + +Routing uses the reference list and mask after applying `sd_cpp_extra_args`. Both `ref_images: []` and `ref_images: null` clear the reference list; `mask_image: null` clears an uploaded mask. An explicit `init_image` in `sd_cpp_extra_args`, including `null`, takes precedence over the implicit init-image selection. + Reference encoding then follows model presets and `ref_image_args`. To skip input geometry for references and disable resizing before VAE encoding, include this in `prompt`: diff --git a/examples/server/openai_edit_policy.h b/examples/server/openai_edit_policy.h new file mode 100644 index 000000000..c963a887f --- /dev/null +++ b/examples/server/openai_edit_policy.h @@ -0,0 +1,44 @@ +#ifndef __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__ +#define __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__ + +#include "common/resource_owners.hpp" + +enum class OpenAIEditInitImageAction { + KeepExisting, + UseUploaded, + Clear, +}; + +inline bool openai_edit_uses_uploaded_image_as_init(bool supports_ref_image_conditioning, + bool has_ref_images, + bool has_mask, + bool has_explicit_strength_override) { + return !supports_ref_image_conditioning || !has_ref_images || has_mask || has_explicit_strength_override; +} + +inline OpenAIEditInitImageAction apply_openai_edit_init_image_policy( + SDImageOwner& init_image, + SDImageOwner& uploaded_image, + bool uploaded_image_was_first_reference, + bool supports_ref_image_conditioning, + bool has_ref_images, + bool has_mask, + bool has_extra_init_image_override, + bool has_explicit_strength_override) { + if (!uploaded_image_was_first_reference || has_extra_init_image_override) { + return OpenAIEditInitImageAction::KeepExisting; + } + + if (openai_edit_uses_uploaded_image_as_init(supports_ref_image_conditioning, + has_ref_images, + has_mask, + has_explicit_strength_override)) { + init_image = std::move(uploaded_image); + return OpenAIEditInitImageAction::UseUploaded; + } + + init_image.reset(); + return OpenAIEditInitImageAction::Clear; +} + +#endif // __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__ diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp index af0cdfaa1..097b04204 100644 --- a/examples/server/routes_openai.cpp +++ b/examples/server/routes_openai.cpp @@ -7,6 +7,7 @@ #include "common/common.h" #include "common/media_io.h" #include "common/resource_owners.hpp" +#include "openai_edit_policy.h" static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) { std::regex re("(.*?)"); @@ -157,7 +158,24 @@ static bool build_openai_edit_request(const httplib::Request& req, request.gen_params.height = height; request.gen_params.batch_count = n; - std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt); + std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt); + const json extra_args = sd_cpp_extra_args_str.empty() + ? json::object() + : json::parse(sd_cpp_extra_args_str, nullptr, false); + const bool has_extra_init_image_override = extra_args.is_object() && extra_args.contains("init_image"); + const bool has_extra_ref_images_override = extra_args.is_object() && extra_args.contains("ref_images"); + const bool has_extra_strength_override = extra_args.is_object() && + extra_args.contains("strength") && + extra_args["strength"].is_number(); + const bool has_extra_mask_image = extra_args.is_object() && + extra_args.contains("mask_image") && + !extra_args["mask_image"].is_null(); + const bool supports_ref_image_conditioning = sd_ctx_supports_ref_image_conditioning(runtime.sd_ctx); + const bool may_need_uploaded_init_image = !has_extra_init_image_override && + (!supports_ref_image_conditioning || !mask_bytes.empty() || + has_extra_mask_image || has_extra_strength_override || has_extra_ref_images_override); + SDImageOwner uploaded_image_for_init; + bool uploaded_image_was_first_reference = false; for (auto& bytes : images_bytes) { int img_w = 0; int img_h = 0; @@ -177,10 +195,9 @@ static bool build_openai_edit_request(const httplib::Request& req, request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height); if (is_first_ref_image) { - request.gen_params.init_image = image_owner; - if (request.gen_params.init_image.get().data == nullptr) { - error_message = "could not allocate init image"; - return false; + uploaded_image_was_first_reference = true; + if (may_need_uploaded_init_image) { + uploaded_image_for_init = image_owner; } } @@ -214,6 +231,21 @@ static bool build_openai_edit_request(const httplib::Request& req, return false; } + const auto init_image_action = apply_openai_edit_init_image_policy( + request.gen_params.init_image, + uploaded_image_for_init, + uploaded_image_was_first_reference, + supports_ref_image_conditioning, + !request.gen_params.ref_images.empty(), + request.gen_params.mask_image.get().data != nullptr, + has_extra_init_image_override, + has_extra_strength_override); + if (init_image_action == OpenAIEditInitImageAction::UseUploaded && + request.gen_params.init_image.get().data == nullptr) { + error_message = "could not allocate init image"; + return false; + } + // Intentionally disable prompt-embedded LoRA tag parsing for server APIs. if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) { error_message = "invalid params"; diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 267e7a7b5..404d8cbdd 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -486,6 +486,8 @@ SD_API int32_t sd_get_num_physical_cores(); SD_API const char* sd_get_system_info(); SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx); SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx); +/** Returns whether the loaded model consumes ref_images as conditioning inputs. */ +SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx); // ControlNet hot-swap APIs are not safe to call while generation is in flight. SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path); diff --git a/src/model.h b/src/model.h index c8d803a66..0c1a37a66 100644 --- a/src/model.h +++ b/src/model.h @@ -365,6 +365,29 @@ static inline bool sd_version_is_control(SDVersion version) { return version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2; } +// These model families consume uploaded reference images as generation +// conditioning. Inpaint and control variants keep their init-image semantics. +static inline bool sd_version_supports_ref_image_conditioning(SDVersion version) { + if (sd_version_is_inpaint(version) || sd_version_is_control(version) || version == VERSION_CHROMA_RADIANCE) { + return false; + } + + return sd_version_is_unet_edit(version) || + sd_version_is_flux(version) || + sd_version_is_flux3(version) || + sd_version_is_flux2(version) || + sd_version_is_qwen_image(version) || + sd_version_is_longcat(version) || + sd_version_is_z_image(version) || + sd_version_is_boogu_image(version) || + sd_version_is_llada_image(version) || + sd_version_is_krea2(version) || + sd_version_is_mage_flow(version) || + sd_version_is_anima(version) || + sd_version_is_sefi_image(version) || + sd_version_is_pid(version); +} + static bool sd_version_is_inpaint_or_unet_edit(SDVersion version) { return sd_version_is_unet_edit(version) || sd_version_is_inpaint(version) || sd_version_is_control(version); } diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 1cc4c09d3..d476f92d6 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -704,6 +704,13 @@ SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) { return sd_version_supports_video_generation(sd_ctx->sd->version); } +SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { + return false; + } + return sd_version_supports_ref_image_conditioning(sd_ctx->sd->version); +} + SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) { if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) { return false;