[BUG] Remove a curl easy handle before releasing what it points at

CHANGELOG.md

@@ -71,6 +71,10 @@ Increment the:
   deprecated C headers (`stdint.h`, `stddef.h`, `stdlib.h`, `string.h`,
   `stdio.h`, `ctype.h`, `limits.h`, `assert.h`) with their C++ equivalents
   ([#4349](https://github.com/open-telemetry/opentelemetry-cpp/pull/4349))
+* [BUG] Remove a curl easy handle from the multi handle before freeing that
+  handle and the header list it points at, and keep both when libcurl will not
+  take the handle back
+  [#4391](https://github.com/open-telemetry/opentelemetry-cpp/issues/4391)
 
 * [CONFIGURATION] Add SDK component builder interfaces to the registry
   [#4358](https://github.com/open-telemetry/opentelemetry-cpp/issues/4358)

ext/include/opentelemetry/ext/http/client/curl/http_client_curl.h

@@ -366,6 +366,9 @@ class HttpClient : public opentelemetry::ext::http::client::HttpClient
   bool doRetrySessions(bool report_all);
   void resetMultiHandle();
 
+  bool detachHandle(CURL *easy_handle);
+  void releaseQuarantinedHandles();
+
   std::mutex multi_handle_m_;
   CURLM *multi_handle_;
   std::atomic<uint64_t> next_session_id_{0};
@@ -376,8 +379,33 @@ class HttpClient : public opentelemetry::ext::http::client::HttpClient
   std::unordered_map<uint64_t, std::shared_ptr<Session>> sessions_;
   std::unordered_set<uint64_t> pending_to_add_session_ids_;
   std::unordered_map<uint64_t, std::shared_ptr<Session>> pending_to_abort_sessions_;
-  std::unordered_map<uint64_t, HttpCurlEasyResource> pending_to_remove_session_handles_;
-  std::list<std::shared_ptr<Session>> pending_to_remove_sessions_;
+  // One easy handle on its way back to libcurl, with the session it still names through
+  // CURLOPT_PRIVATE. The session is taken when the record is made, so that nothing has to find
+  // it again later against a sessions_ the calling thread is free to change in between.
+  //
+  // There is one of these per handle rather than per session, because a session that starts
+  // another request hands over a second handle while the first is still queued, and one entry
+  // per session would drop the first one, leaving its easy handle and header list with no owner.
+  struct PendingCurlRemoval
+  {
+    uint64_t session_id;
+    HttpCurlEasyResource resource;
+    std::shared_ptr<Session> owner;
+  };
+
+  std::list<PendingCurlRemoval> pending_to_remove_session_handles_;
+
+  // Which easy handles the multi handle holds, recorded when curl_multi_add_handle accepts one.
+  // What curl_multi_remove_handle returns cannot answer that question afterwards: it reports
+  // whether the removal succeeded, and libcurl 8.10 and 8.11 reject a handle the multi handle
+  // does not hold where every other version accepts it. Background thread only.
+  std::unordered_set<CURL *> attached_handles_;
+
+  // Handles libcurl would not give back. A transfer may still be running on one, and its
+  // CURLOPT_PRIVATE names the session, so freeing either would leave libcurl and this client
+  // reading storage that has been released. Both are held until curl_multi_cleanup detaches
+  // the handle, which is where they are freed.
+  std::list<PendingCurlRemoval> quarantined_handles_;
   std::deque<std::shared_ptr<Session>> pending_to_retry_sessions_;
 
   std::mutex background_thread_m_;

ext/include/opentelemetry/ext/http/client/curl/http_operation_curl.h

@@ -285,7 +285,12 @@ class HttpOperation
    *
    * @param code CURLcode
    */
-  void PerformCurlMessage(CURLcode code);
+  /**
+   * Read one completed curl message.
+   * @return true when the operation has been rewound for another attempt, and the caller is to
+   * take it on. False when the message was not this operation's to read, or it is finished.
+   */
+  bool PerformCurlMessage(CURLcode code);
 
   inline CURL *GetCurlEasyHandle() noexcept { return curl_resource_.easy_handle; }