fix: carry the pickle line-span fix for legacy checkpoints above 2 GiB

AGENTS.md

@@ -8,7 +8,8 @@ Guidance for coding agents working in `stable-diffusion-native`.
   consumed by `llamadart`. The Dart API and model handling live in `llamadart`.
 - Upstream is the `third_party/stable-diffusion.cpp` submodule. Update the
   submodule pin instead of patching vendored sources; report upstream bugs
-  there.
+  there. Never edit the submodule. A fix the pin lacks is carried only as a
+  `patches/` entry (`patches/README.md`) with a test that fails without it.
 - What upstream's C API lacks goes in `src/`, which the top-level
   `CMakeLists.txt` compiles into the same library. A wrapper export is an
   `sd_dart_*` function declared with `SD_API` in `src/sd_dart_wrapper.h`, with

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.13)
+cmake_minimum_required(VERSION 3.18)
 project(stable_diffusion_native LANGUAGES C CXX)
 
 set(SD_NATIVE_UPSTREAM_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party/stable-diffusion.cpp")
@@ -17,6 +17,66 @@ define_property(DIRECTORY PROPERTY EP_PREFIX INHERITED
 set_property(DIRECTORY PROPERTY EP_PREFIX "${CMAKE_BINARY_DIR}/ep")
 add_subdirectory("${SD_NATIVE_UPSTREAM_DIR}" sd)
 
+# Fixes the pinned upstream lacks (patches/README.md). Each patch is applied to
+# build-tree copies of the upstream sources it changes, and the library
+# compiles the copies, so the submodule stays pristine. Configuring fails when
+# a patch no longer applies or changes nothing.
+find_package(Git REQUIRED)
+function(sd_native_carry_patch patch)
+    get_filename_component(name "${patch}" NAME_WE)
+    set(patched_root "${CMAKE_BINARY_DIR}/patched")
+    set(patched_dir "${patched_root}/${name}")
+    file(REMOVE_RECURSE "${patched_dir}")
+    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS
+        "${CMAKE_CURRENT_SOURCE_DIR}/patches/${patch}")
+    get_target_property(sources stable-diffusion SOURCES)
+    foreach(path IN LISTS ARGN)
+        set(upstream_file "${SD_NATIVE_UPSTREAM_DIR}/${path}")
+        if(NOT "${upstream_file}" IN_LIST sources)
+            message(FATAL_ERROR "patches/${patch}: ${path} is not a stable-diffusion source")
+        endif()
+        file(READ "${upstream_file}" text)
+        # A Windows checkout can convert upstream's line endings; patches use LF.
+        string(REPLACE "\r\n" "\n" text "${text}")
+        file(WRITE "${patched_dir}/${path}" "${text}")
+    endforeach()
+
+    # The build tree can sit inside this repository's checkout, where git apply
+    # would skip every path outside it and still succeed.
+    get_filename_component(ceiling "${patched_root}" REALPATH)
+    execute_process(
+        COMMAND "${CMAKE_COMMAND}" -E env "GIT_CEILING_DIRECTORIES=${ceiling}"
+                "${GIT_EXECUTABLE}" apply --verbose "${CMAKE_CURRENT_SOURCE_DIR}/patches/${patch}"
+        WORKING_DIRECTORY "${patched_dir}"
+        RESULT_VARIABLE result
+        OUTPUT_VARIABLE output
+        ERROR_VARIABLE output)
+    if(NOT result EQUAL 0)
+        message(FATAL_ERROR "patches/${patch} does not apply to the pinned upstream:\n${output}")
+    endif()
+
+    foreach(path IN LISTS ARGN)
+        set(upstream_file "${SD_NATIVE_UPSTREAM_DIR}/${path}")
+        set(patched_file "${patched_dir}/${path}")
+        file(READ "${upstream_file}" original)
+        string(REPLACE "\r\n" "\n" original "${original}")
+        file(READ "${patched_file}" patched)
+        if(original STREQUAL patched)
+            message(FATAL_ERROR "patches/${patch} left ${path} unchanged:\n${output}")
+        endif()
+        list(REMOVE_ITEM sources "${upstream_file}")
+        list(APPEND sources "${patched_file}")
+        # Quoted includes in the copy must still find the upstream files next to it.
+        get_filename_component(upstream_file_dir "${upstream_file}" DIRECTORY)
+        set_source_files_properties("${patched_file}" TARGET_DIRECTORY stable-diffusion
+            PROPERTIES INCLUDE_DIRECTORIES "${upstream_file_dir}")
+    endforeach()
+    set_property(TARGET stable-diffusion PROPERTY SOURCES ${sources})
+    message(STATUS "Carrying patches/${patch}")
+endfunction()
+
+sd_native_carry_patch(pickle_line_span.patch src/model_io/pickle_io.cpp)
+
 # Wrapper exports are compiled into the upstream library target, so they share
 # its SD_API export macro and reach upstream without patching it.
 target_sources(stable-diffusion PRIVATE

README.md

@@ -11,7 +11,8 @@ consumed by [llamadart](https://github.com/leehack/llamadart) and other
 wrappers. It stays independent from `llamadart` so the artifacts can be reused
 elsewhere.
 
-- Pin an upstream stable-diffusion.cpp commit as a submodule.
+- Pin an upstream stable-diffusion.cpp commit as a submodule, plus the few
+  [carried fixes](patches/README.md) the pin lacks.
 - Build one shared library per platform with upstream's C API
   (`stable-diffusion.h`) as the FFI boundary.
 - Export only that API and the [wrapper API](#wrapper-api) this repository

patches/README.md

@@ -0,0 +1,15 @@
+# Carried upstream patches
+
+Fixes the pinned stable-diffusion.cpp lacks, carried until a pin includes them.
+`CMakeLists.txt` applies each patch to build-tree copies of the upstream
+sources it names and compiles the copies, so the submodule stays pristine.
+Configuring fails when a patch no longer applies or changes nothing; then drop
+or rebase it. Build info and the release manifest list the patches next to the
+upstream commit.
+
+A patch here needs a regression test that fails against the unpatched pin and
+an upstream issue or pull request to retire it.
+
+| Patch | Sources | Fixes | Test |
+| --- | --- | --- | --- |
+| `pickle_line_span.patch` | `src/model_io/pickle_io.cpp` | A legacy PyTorch checkpoint above 2 GiB is rejected: the pickle scanner narrows the rest of the file to `int` before looking for a newline ([#23](https://github.com/leehack/stable-diffusion-native/issues/23)). | `tests/test_pickle_span.py` |

patches/pickle_line_span.patch

@@ -0,0 +1,134 @@
+diff --git a/src/model_io/pickle_io.cpp b/src/model_io/pickle_io.cpp
+--- a/src/model_io/pickle_io.cpp
++++ b/src/model_io/pickle_io.cpp
+@@ -1,5 +1,7 @@
+ #include "pickle_io.h"
+
++#include <algorithm>
++#include <cstddef>
+ #include <cstdlib>
+ #include <cstring>
+ #include <limits>
+@@ -115,6 +117,13 @@ static void set_error(std::string* error, const std::string& message) {
+     }
+ }
+
++// A legacy checkpoint passes the rest of the file, tensor storage included, so
++// the span can exceed INT_MAX. No pickle line is that long.
++static int find_line_end(const uint8_t* p, const uint8_t* end) {
++    const ptrdiff_t max_len = std::numeric_limits<int>::max();
++    return find_char(p, (int)std::min(end - p, max_len), '\n');
++}
++
+ bool skip_pickle_object(const uint8_t* buffer, size_t buffer_size, size_t* object_size) {
+     const uint8_t* p   = buffer;
+     const uint8_t* end = buffer + buffer_size;
+@@ -148,7 +157,7 @@ bool skip_pickle_object(const uint8_t* buffer, size_t buffer_size, size_t* objec
+             case 'F':    // FLOAT            = b'F'   # push decimal float line
+             case 'S':    // STRING           = b'S'   # push quoted string line
+             case 'V': {  // UNICODE          = b'V'   # push raw-unicode string line
+-                int len = find_char(p, (int)(end - p), '\n');
++                int len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+@@ -206,7 +215,7 @@ bool skip_pickle_object(const uint8_t* buffer, size_t buffer_size, size_t* objec
+                 p += 1 + p[0];
+                 break;
+             case 'P': {  // PERSID           = b'P'   # persistent id, newline-terminated
+-                int len = find_char(p, (int)(end - p), '\n');
++                int len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+@@ -216,12 +225,12 @@ bool skip_pickle_object(const uint8_t* buffer, size_t buffer_size, size_t* objec
+                 p += 8;
+                 break;
+             case 'c': {  // GLOBAL           = b'c'   # push module/name global reference
+-                int len = find_char(p, (int)(end - p), '\n');
++                int len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+                 p += len + 1;
+-                len = find_char(p, (int)(end - p), '\n');
++                len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+@@ -261,12 +270,12 @@ bool skip_pickle_object(const uint8_t* buffer, size_t buffer_size, size_t* objec
+             case 'o':   // OBJ              = b'o'   # build class instance from mark
+                 break;
+             case 'i': {  // INST             = b'i'   # build class instance from module/name
+-                int len = find_char(p, (int)(end - p), '\n');
++                int len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+                 p += len + 1;
+-                len = find_char(p, (int)(end - p), '\n');
++                len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+@@ -680,7 +689,7 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
+                 p += 4;
+                 break;
+             case 'I': {  // INT              = b'I'   # push decimal integer line
+-                int len = find_char(p, (int)(end - p), '\n');
++                int len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+@@ -695,7 +704,7 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
+                 }
+             } break;
+             case 'L': {  // LONG             = b'L'   # push decimal long integer line
+-                int len = find_char(p, (int)(end - p), '\n');
++                int len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+@@ -707,7 +716,7 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
+                 stack.push_back(make_int_value(std::strtoll(s.c_str(), nullptr, 10)));
+             } break;
+             case 'F': {  // FLOAT            = b'F'   # push decimal float line
+-                int len = find_char(p, (int)(end - p), '\n');
++                int len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+@@ -799,7 +808,7 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
+                 p += len;
+             } break;
+             case 'S': {  // STRING           = b'S'   # push quoted string line
+-                int len = find_char(p, (int)(end - p), '\n');
++                int len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+@@ -811,7 +820,7 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
+                 stack.push_back(make_string_value(s));
+             } break;
+             case 'V': {  // UNICODE          = b'V'   # push raw-unicode string line
+-                int len = find_char(p, (int)(end - p), '\n');
++                int len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+@@ -819,13 +828,13 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer,
+                 p += len + 1;
+             } break;
+             case 'c': {  // GLOBAL           = b'c'   # push module/name global reference
+-                int len = find_char(p, (int)(end - p), '\n');
++                int len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }
+                 std::string module((const char*)p, len);
+                 p += len + 1;
+-                len = find_char(p, (int)(end - p), '\n');
++                len = find_line_end(p, end);
+                 if (len < 0) {
+                     return false;
+                 }