From c62a3be5558129a91da910f7de9f5328ac5f96f4 Mon Sep 17 00:00:00 2001 From: losewayy <104015127+losewayy@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:18:49 +0800 Subject: [PATCH 1/3] fix: route reference edits through conditioning --- examples/server/api.md | 8 ++++++ examples/server/openai_edit_policy.h | 41 ++++++++++++++++++++++++++++ examples/server/routes_openai.cpp | 40 +++++++++++++++++++++++---- include/stable-diffusion.h | 2 ++ src/model.h | 24 ++++++++++++++++ src/stable-diffusion.cpp | 7 +++++ 6 files changed, 117 insertions(+), 5 deletions(-) create mode 100644 examples/server/openai_edit_policy.h diff --git a/examples/server/api.md b/examples/server/api.md index cff4e84559..7ee0d11ffa 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -155,6 +155,14 @@ image establishes the generation dimensions if `size` is omitted. Input geometry follows `image_preprocess`: references preserve their dimensions by default, while init and mask use the generation canvas preset. +When the loaded model supports reference-image conditioning and no mask is +provided, uploaded images are used as references instead of initializing the +img2img latent. Other models continue to use the first uploaded image as the +img2img init image. With a mask, the first uploaded image is also used as the +init image for inpainting. An explicit `init_image` in `sd_cpp_extra_args` +continues to override this routing. An explicit `strength` in `sd_cpp_extra_args` +also keeps the existing img2img strength behavior. + Reference encoding then follows model presets and `ref_image_args`. To skip input geometry for references and disable resizing before VAE encoding, include this in `prompt`: diff --git a/examples/server/openai_edit_policy.h b/examples/server/openai_edit_policy.h new file mode 100644 index 0000000000..bc19017cb4 --- /dev/null +++ b/examples/server/openai_edit_policy.h @@ -0,0 +1,41 @@ +#ifndef __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__ +#define __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__ + +#include "common/resource_owners.hpp" + +enum class OpenAIEditInitImageAction { + KeepExisting, + UseUploaded, + Clear, +}; + +inline bool openai_edit_uses_uploaded_image_as_init(bool supports_ref_image_conditioning, + bool has_mask, + bool has_explicit_strength_override) { + return !supports_ref_image_conditioning || has_mask || has_explicit_strength_override; +} + +inline OpenAIEditInitImageAction apply_openai_edit_init_image_policy( + SDImageOwner& init_image, + SDImageOwner& uploaded_image, + bool uploaded_image_was_first_reference, + bool supports_ref_image_conditioning, + bool has_mask, + bool has_extra_init_image_override, + bool has_explicit_strength_override) { + if (!uploaded_image_was_first_reference || has_extra_init_image_override) { + return OpenAIEditInitImageAction::KeepExisting; + } + + if (openai_edit_uses_uploaded_image_as_init(supports_ref_image_conditioning, + has_mask, + has_explicit_strength_override)) { + init_image = std::move(uploaded_image); + return OpenAIEditInitImageAction::UseUploaded; + } + + init_image.reset(); + return OpenAIEditInitImageAction::Clear; +} + +#endif // __SD_EXAMPLES_SERVER_OPENAI_EDIT_POLICY_H__ diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp index af0cdfaa17..412715d5e6 100644 --- a/examples/server/routes_openai.cpp +++ b/examples/server/routes_openai.cpp @@ -7,6 +7,7 @@ #include "common/common.h" #include "common/media_io.h" #include "common/resource_owners.hpp" +#include "openai_edit_policy.h" static std::string extract_and_remove_sd_cpp_extra_args(std::string& text) { std::regex re("(.*?)"); @@ -157,7 +158,23 @@ static bool build_openai_edit_request(const httplib::Request& req, request.gen_params.height = height; request.gen_params.batch_count = n; - std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt); + std::string sd_cpp_extra_args_str = extract_and_remove_sd_cpp_extra_args(request.gen_params.prompt); + const json extra_args = sd_cpp_extra_args_str.empty() + ? json::object() + : json::parse(sd_cpp_extra_args_str, nullptr, false); + const bool has_extra_init_image_override = extra_args.is_object() && extra_args.contains("init_image"); + const bool has_extra_strength_override = extra_args.is_object() && + extra_args.contains("strength") && + extra_args["strength"].is_number(); + const bool has_extra_mask_image = extra_args.is_object() && + extra_args.contains("mask_image") && + !extra_args["mask_image"].is_null(); + const bool supports_ref_image_conditioning = sd_ctx_supports_ref_image_conditioning(runtime.sd_ctx); + const bool may_need_uploaded_init_image = !has_extra_init_image_override && + (!supports_ref_image_conditioning || !mask_bytes.empty() || + has_extra_mask_image || has_extra_strength_override); + SDImageOwner uploaded_image_for_init; + bool uploaded_image_was_first_reference = false; for (auto& bytes : images_bytes) { int img_w = 0; int img_h = 0; @@ -177,10 +194,9 @@ static bool build_openai_edit_request(const httplib::Request& req, request.gen_params.set_width_and_height_if_unset(image_owner.get().width, image_owner.get().height); if (is_first_ref_image) { - request.gen_params.init_image = image_owner; - if (request.gen_params.init_image.get().data == nullptr) { - error_message = "could not allocate init image"; - return false; + uploaded_image_was_first_reference = true; + if (may_need_uploaded_init_image) { + uploaded_image_for_init = image_owner; } } @@ -214,6 +230,20 @@ static bool build_openai_edit_request(const httplib::Request& req, return false; } + const auto init_image_action = apply_openai_edit_init_image_policy( + request.gen_params.init_image, + uploaded_image_for_init, + uploaded_image_was_first_reference, + supports_ref_image_conditioning, + !mask_bytes.empty() || request.gen_params.mask_image.get().data != nullptr, + has_extra_init_image_override, + has_extra_strength_override); + if (init_image_action == OpenAIEditInitImageAction::UseUploaded && + request.gen_params.init_image.get().data == nullptr) { + error_message = "could not allocate init image"; + return false; + } + // Intentionally disable prompt-embedded LoRA tag parsing for server APIs. if (!request.gen_params.resolve_and_validate(IMG_GEN, "", runtime.ctx_params->hires_upscalers_dir, true)) { error_message = "invalid params"; diff --git a/include/stable-diffusion.h b/include/stable-diffusion.h index 267e7a7b55..404d8cbddc 100644 --- a/include/stable-diffusion.h +++ b/include/stable-diffusion.h @@ -486,6 +486,8 @@ SD_API int32_t sd_get_num_physical_cores(); SD_API const char* sd_get_system_info(); SD_API bool sd_ctx_supports_image_generation(const sd_ctx_t* sd_ctx); SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx); +/** Returns whether the loaded model consumes ref_images as conditioning inputs. */ +SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx); // ControlNet hot-swap APIs are not safe to call while generation is in flight. SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path); diff --git a/src/model.h b/src/model.h index c8d803a667..9f0c6648a6 100644 --- a/src/model.h +++ b/src/model.h @@ -365,6 +365,30 @@ static inline bool sd_version_is_control(SDVersion version) { return version == VERSION_FLUX_CONTROLS || version == VERSION_FLEX_2; } +// These model families consume uploaded reference images as generation +// conditioning. Inpaint and control variants keep their init-image semantics. +static inline bool sd_version_supports_ref_image_conditioning(SDVersion version) { + if (sd_version_is_inpaint(version) || sd_version_is_control(version)) { + return false; + } + + return sd_version_is_unet_edit(version) || + sd_version_is_flux(version) || + sd_version_is_flux3(version) || + sd_version_is_flux2(version) || + sd_version_is_qwen_image(version) || + sd_version_is_longcat(version) || + sd_version_is_z_image(version) || + sd_version_is_boogu_image(version) || + sd_version_is_llada_image(version) || + sd_version_is_krea2(version) || + sd_version_is_mage_flow(version) || + sd_version_is_anima(version) || + sd_version_is_sefi_image(version) || + sd_version_is_pid(version) || + version == VERSION_HIDREAM_O1; +} + static bool sd_version_is_inpaint_or_unet_edit(SDVersion version) { return sd_version_is_unet_edit(version) || sd_version_is_inpaint(version) || sd_version_is_control(version); } diff --git a/src/stable-diffusion.cpp b/src/stable-diffusion.cpp index 1cc4c09d3d..d476f92d6b 100644 --- a/src/stable-diffusion.cpp +++ b/src/stable-diffusion.cpp @@ -704,6 +704,13 @@ SD_API bool sd_ctx_supports_video_generation(const sd_ctx_t* sd_ctx) { return sd_version_supports_video_generation(sd_ctx->sd->version); } +SD_API bool sd_ctx_supports_ref_image_conditioning(const sd_ctx_t* sd_ctx) { + if (sd_ctx == nullptr || sd_ctx->sd == nullptr) { + return false; + } + return sd_version_supports_ref_image_conditioning(sd_ctx->sd->version); +} + SD_API bool sd_ctx_load_control_net(sd_ctx_t* sd_ctx, const char* path) { if (sd_ctx == nullptr || sd_ctx->sd == nullptr || path == nullptr) { return false; From 401b5381d0e5dfcb326e612187f29d9a1ffd0cdf Mon Sep 17 00:00:00 2001 From: losewayy <104015127+losewayy@users.noreply.github.com> Date: Mon, 19 Oct 2026 15:46:07 +0800 Subject: [PATCH 2/3] fix: keep HiDream O1 on the existing init-image path --- src/model.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/model.h b/src/model.h index 9f0c6648a6..800a98d550 100644 --- a/src/model.h +++ b/src/model.h @@ -385,8 +385,7 @@ static inline bool sd_version_supports_ref_image_conditioning(SDVersion version) sd_version_is_mage_flow(version) || sd_version_is_anima(version) || sd_version_is_sefi_image(version) || - sd_version_is_pid(version) || - version == VERSION_HIDREAM_O1; + sd_version_is_pid(version); } static bool sd_version_is_inpaint_or_unet_edit(SDVersion version) { From d85b57bcc419aa472318d8aecd1d89ba9c458f95 Mon Sep 17 00:00:00 2001 From: losewayy <104015127+losewayy@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:47:32 +0800 Subject: [PATCH 3/3] fix: preserve edit routing after argument overrides --- examples/server/api.md | 10 +++------- examples/server/openai_edit_policy.h | 5 ++++- examples/server/routes_openai.cpp | 6 ++++-- src/model.h | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/examples/server/api.md b/examples/server/api.md index 7ee0d11ffa..f2f6d32b05 100644 --- a/examples/server/api.md +++ b/examples/server/api.md @@ -155,13 +155,9 @@ image establishes the generation dimensions if `size` is omitted. Input geometry follows `image_preprocess`: references preserve their dimensions by default, while init and mask use the generation canvas preset. -When the loaded model supports reference-image conditioning and no mask is -provided, uploaded images are used as references instead of initializing the -img2img latent. Other models continue to use the first uploaded image as the -img2img init image. With a mask, the first uploaded image is also used as the -init image for inpainting. An explicit `init_image` in `sd_cpp_extra_args` -continues to override this routing. An explicit `strength` in `sd_cpp_extra_args` -also keeps the existing img2img strength behavior. +When the request starts without preconfigured reference images, the first uploaded image becomes the img2img init image if the loaded model does not support reference-image conditioning, the final reference list is empty, a mask is present, or an explicit numeric `strength` override is provided. Otherwise, only the final reference list is used for image conditioning, without initializing the img2img latent. Requests with preconfigured references retain their existing init-image selection. + +Routing uses the reference list and mask after applying `sd_cpp_extra_args`. Both `ref_images: []` and `ref_images: null` clear the reference list; `mask_image: null` clears an uploaded mask. An explicit `init_image` in `sd_cpp_extra_args`, including `null`, takes precedence over the implicit init-image selection. Reference encoding then follows model presets and `ref_image_args`. To skip input geometry for references and disable resizing before VAE encoding, include diff --git a/examples/server/openai_edit_policy.h b/examples/server/openai_edit_policy.h index bc19017cb4..c963a887f8 100644 --- a/examples/server/openai_edit_policy.h +++ b/examples/server/openai_edit_policy.h @@ -10,9 +10,10 @@ enum class OpenAIEditInitImageAction { }; inline bool openai_edit_uses_uploaded_image_as_init(bool supports_ref_image_conditioning, + bool has_ref_images, bool has_mask, bool has_explicit_strength_override) { - return !supports_ref_image_conditioning || has_mask || has_explicit_strength_override; + return !supports_ref_image_conditioning || !has_ref_images || has_mask || has_explicit_strength_override; } inline OpenAIEditInitImageAction apply_openai_edit_init_image_policy( @@ -20,6 +21,7 @@ inline OpenAIEditInitImageAction apply_openai_edit_init_image_policy( SDImageOwner& uploaded_image, bool uploaded_image_was_first_reference, bool supports_ref_image_conditioning, + bool has_ref_images, bool has_mask, bool has_extra_init_image_override, bool has_explicit_strength_override) { @@ -28,6 +30,7 @@ inline OpenAIEditInitImageAction apply_openai_edit_init_image_policy( } if (openai_edit_uses_uploaded_image_as_init(supports_ref_image_conditioning, + has_ref_images, has_mask, has_explicit_strength_override)) { init_image = std::move(uploaded_image); diff --git a/examples/server/routes_openai.cpp b/examples/server/routes_openai.cpp index 412715d5e6..097b042040 100644 --- a/examples/server/routes_openai.cpp +++ b/examples/server/routes_openai.cpp @@ -163,6 +163,7 @@ static bool build_openai_edit_request(const httplib::Request& req, ? json::object() : json::parse(sd_cpp_extra_args_str, nullptr, false); const bool has_extra_init_image_override = extra_args.is_object() && extra_args.contains("init_image"); + const bool has_extra_ref_images_override = extra_args.is_object() && extra_args.contains("ref_images"); const bool has_extra_strength_override = extra_args.is_object() && extra_args.contains("strength") && extra_args["strength"].is_number(); @@ -172,7 +173,7 @@ static bool build_openai_edit_request(const httplib::Request& req, const bool supports_ref_image_conditioning = sd_ctx_supports_ref_image_conditioning(runtime.sd_ctx); const bool may_need_uploaded_init_image = !has_extra_init_image_override && (!supports_ref_image_conditioning || !mask_bytes.empty() || - has_extra_mask_image || has_extra_strength_override); + has_extra_mask_image || has_extra_strength_override || has_extra_ref_images_override); SDImageOwner uploaded_image_for_init; bool uploaded_image_was_first_reference = false; for (auto& bytes : images_bytes) { @@ -235,7 +236,8 @@ static bool build_openai_edit_request(const httplib::Request& req, uploaded_image_for_init, uploaded_image_was_first_reference, supports_ref_image_conditioning, - !mask_bytes.empty() || request.gen_params.mask_image.get().data != nullptr, + !request.gen_params.ref_images.empty(), + request.gen_params.mask_image.get().data != nullptr, has_extra_init_image_override, has_extra_strength_override); if (init_image_action == OpenAIEditInitImageAction::UseUploaded && diff --git a/src/model.h b/src/model.h index 800a98d550..0c1a37a66c 100644 --- a/src/model.h +++ b/src/model.h @@ -368,7 +368,7 @@ static inline bool sd_version_is_control(SDVersion version) { // These model families consume uploaded reference images as generation // conditioning. Inpaint and control variants keep their init-image semantics. static inline bool sd_version_supports_ref_image_conditioning(SDVersion version) { - if (sd_version_is_inpaint(version) || sd_version_is_control(version)) { + if (sd_version_is_inpaint(version) || sd_version_is_control(version) || version == VERSION_CHROMA_RADIANCE) { return false; }