feat(fs): add lightweight object-store and S3 filesystem support

.github/workflows/build_and_test.yaml

@@ -71,6 +71,10 @@ jobs:
         if: ${{ !matrix.skip_rust }}
         shell: bash
         run: ci/scripts/setup_rust.sh
+      - name: Install HTTP and TLS development dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libcurl4-openssl-dev libssl-dev
       - name: Build Paimon
         shell: bash
         env:

.github/workflows/gcc8_test.yaml

@@ -42,7 +42,7 @@ jobs:
       - name: Install dependencies
         run: |
           apt-get update
-          DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git git-lfs tar curl tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
+          DEBIAN_FRONTEND=noninteractive apt-get install -y gcc-8 g++-8 ninja-build git git-lfs tar curl libcurl4-openssl-dev libssl-dev tzdata zip unzip pkg-config build-essential python3-dev gdb sudo
           curl -L -O https://github.com/Kitware/CMake/releases/download/v3.28.3/cmake-3.28.3-linux-x86_64.tar.gz
           tar -zxvf cmake-3.28.3-linux-x86_64.tar.gz -C /usr/local --strip-components=1
           rm cmake-3.28.3-linux-x86_64.tar.gz

CMakeLists.txt

@@ -56,6 +56,7 @@ option(PAIMON_USE_CXX11_ABI "Use C++11 ABI" ON)
 option(PAIMON_ENABLE_AVRO "Whether to enable avro file format" ON)
 option(PAIMON_ENABLE_ORC "Whether to enable orc file format" ON)
 option(PAIMON_ENABLE_JINDO "Whether to enable jindo file system" OFF)
+option(PAIMON_ENABLE_S3 "Whether to enable S3 file system" OFF)
 option(PAIMON_ENABLE_NETWORK_TESTS
        "Whether to enable tests that access real remote services over the network" OFF)
 option(PAIMON_ENABLE_LUMINA "Whether to enable lumina vector index" OFF)
@@ -71,6 +72,9 @@ endif()
 if(PAIMON_ENABLE_JINDO)
     add_definitions(-DPAIMON_ENABLE_JINDO)
 endif()
+if(PAIMON_ENABLE_S3)
+    add_definitions(-DPAIMON_ENABLE_S3)
+endif()
 if(PAIMON_ENABLE_NETWORK_TESTS)
     add_definitions(-DPAIMON_ENABLE_NETWORK_TESTS)
 endif()
@@ -426,6 +430,13 @@ if(PAIMON_BUILD_TESTS)
                                            paimon_jindo_file_system_shared)
         list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
     endif()
+    if(PAIMON_ENABLE_S3)
+        paimon_link_libraries_whole_archive(PAIMON_S3_FILE_SYSTEM_STATIC_LINK_LIBS
+                                            paimon_s3_file_system_static)
+        paimon_link_libraries_no_as_needed(TEST_PLUGIN_LINK_LIBS
+                                           paimon_s3_file_system_shared)
+        list(APPEND TEST_STATIC_LINK_LIBS ${TEST_PLUGIN_LINK_LIBS})
+    endif()
     if(PAIMON_ENABLE_LUMINA)
         paimon_link_libraries_whole_archive(PAIMON_LUMINA_INDEX_STATIC_LINK_LIBS
                                             paimon_lumina_index_static)
@@ -473,9 +484,13 @@ install(FILES "${CMAKE_CURRENT_BINARY_DIR}/PaimonConfig.cmake"
 
 config_summary_message()
 
+if(PAIMON_ENABLE_S3)
+    find_package(CURL REQUIRED)
+endif()
 add_subdirectory(src/paimon)
 add_subdirectory(src/paimon/fs/local)
 add_subdirectory(src/paimon/fs/jindo)
+add_subdirectory(src/paimon/fs/s3)
 add_subdirectory(src/paimon/format/blob)
 add_subdirectory(src/paimon/format/orc)
 add_subdirectory(src/paimon/format/parquet)

ci/scripts/build_paimon.sh

@@ -126,6 +126,7 @@ CMAKE_ARGS=(
     "-DCMAKE_BUILD_TYPE=${build_type}"
     "-DPAIMON_BUILD_TESTS=ON"
     "-DPAIMON_ENABLE_JINDO=ON"
+    "-DPAIMON_ENABLE_S3=ON"
     "-DPAIMON_ENABLE_LUMINA=${ENABLE_LUMINA}"
     "-DPAIMON_ENABLE_LUCENE=ON"
     "-DPAIMON_ENABLE_TANTIVY=${ENABLE_TANTIVY}"

cmake_modules/BuildAwsAuth.cmake

@@ -0,0 +1,193 @@
+# Copyright 2026-present Alibaba Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include(ExternalProject)
+include(GNUInstallDirs)
+
+function(paimon_add_s2n_project)
+    if(DEFINED ENV{PAIMON_AWS_S2N_URL})
+        set(S2N_URL "$ENV{PAIMON_AWS_S2N_URL}")
+    elseif(EXISTS "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}")
+        set(S2N_URL "${THIRDPARTY_DIR}/${PAIMON_AWS_S2N_PKG_NAME}")
+    else()
+        set(S2N_URL
+            "${THIRDPARTY_MIRROR_URL}https://github.com/aws/s2n-tls/archive/refs/tags/${PAIMON_AWS_S2N_BUILD_VERSION}.zip"
+        )
+    endif()
+    set(S2N_C_FLAGS "${EP_C_FLAGS}")
+    set(S2N_CXX_FLAGS "${EP_CXX_FLAGS}")
+    string(REPLACE "-Wdocumentation" "" S2N_C_FLAGS "${S2N_C_FLAGS}")
+    string(REPLACE "-Wdocumentation" "" S2N_CXX_FLAGS "${S2N_CXX_FLAGS}")
+    externalproject_add(s2n_ep
+                        URL ${S2N_URL}
+                        URL_HASH SHA256=${PAIMON_AWS_S2N_BUILD_SHA256_CHECKSUM}
+                        DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+                        CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS}
+                                   -DCMAKE_C_FLAGS=${S2N_C_FLAGS}
+                                   -DCMAKE_CXX_FLAGS=${S2N_CXX_FLAGS}
+                                   -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                                   -DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX}
+                                   -DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR}
+                                   -DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX}
+                                   -Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
+                                   -Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}
+                                   -DBUILD_SHARED_LIBS=OFF
+                                   -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+                                   -DBUILD_TESTING=OFF
+                                   -DS2N_INTERN_LIBCRYPTO=OFF
+                        BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/libs2n.a
+                                         ${THIRDPARTY_LOG_OPTIONS})
+endfunction()
+
+function(paimon_add_aws_c_project
+         NAME
+         VERSION
+         CHECKSUM
+         DEPENDS)
+    string(REPLACE "aws-c-" "AWS_C_" URL_NAME "${NAME}")
+    string(TOUPPER "${URL_NAME}" URL_NAME)
+    set(URL_VAR "PAIMON_${URL_NAME}_URL")
+    set(PKG_NAME_VAR "PAIMON_${URL_NAME}_PKG_NAME")
+    if(DEFINED ENV{${URL_VAR}})
+        set(URL "$ENV{${URL_VAR}}")
+    elseif(EXISTS "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}")
+        set(URL "${THIRDPARTY_DIR}/${${PKG_NAME_VAR}}")
+    else()
+        set(URL
+            "${THIRDPARTY_MIRROR_URL}https://github.com/awslabs/${NAME}/archive/refs/tags/${VERSION}.tar.gz"
+        )
+    endif()
+    set(AWS_PLATFORM_CMAKE_ARGS)
+    list(APPEND
+         AWS_PLATFORM_CMAKE_ARGS
+         -DOPENSSL_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
+         -DOPENSSL_SSL_LIBRARY=${OPENSSL_SSL_LIBRARY}
+         -DOPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}
+         -Dcrypto_INCLUDE_DIR=${OPENSSL_INCLUDE_DIR}
+         -Dcrypto_LIBRARY=${OPENSSL_CRYPTO_LIBRARY})
+    if(NAME STREQUAL "aws-c-cal")
+        list(APPEND AWS_PLATFORM_CMAKE_ARGS -DUSE_OPENSSL=ON)
+    endif()
+    if(APPLE)
+        list(APPEND AWS_PLATFORM_CMAKE_ARGS -DAWS_USE_SECITEM=ON)
+    endif()
+    # aws-c-io's installed package unconditionally looks for s2n on Unix even
+    # when it was built with Apple's native TLS backend.
+    if(APPLE AND (NAME STREQUAL "aws-c-http" OR NAME STREQUAL "aws-c-auth"))
+        list(APPEND AWS_PLATFORM_CMAKE_ARGS -DBYO_CRYPTO=ON)
+    endif()
+    externalproject_add(${NAME}_ep
+                        URL ${URL}
+                        URL_HASH SHA256=${CHECKSUM}
+                        DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+                        CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS}
+                                   -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+                                   -DCMAKE_INSTALL_PREFIX=${AWS_AUTH_PREFIX}
+                                   -DCMAKE_INSTALL_LIBDIR=${AWS_AUTH_INSTALL_LIBDIR}
+                                   -DCMAKE_PREFIX_PATH=${AWS_AUTH_PREFIX}
+                                   -DBUILD_SHARED_LIBS=OFF
+                                   -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+                                   -DBUILD_TESTING=OFF
+                                   -DENABLE_TESTING=OFF
+                                   ${AWS_PLATFORM_CMAKE_ARGS}
+                        DEPENDS ${DEPENDS}
+                        BUILD_BYPRODUCTS ${AWS_AUTH_LIB_DIR}/lib${NAME}.a
+                                         ${THIRDPARTY_LOG_OPTIONS})
+endfunction()
+
+function(build_aws_auth)
+    set(AWS_AUTH_PREFIX
+        "${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install"
+        PARENT_SCOPE)
+    set(AWS_AUTH_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws-auth_ep-install")
+    set(AWS_AUTH_INSTALL_LIBDIR "${CMAKE_INSTALL_LIBDIR}")
+    set(AWS_AUTH_LIB_DIR "${AWS_AUTH_PREFIX}/${AWS_AUTH_INSTALL_LIBDIR}")
+    file(MAKE_DIRECTORY "${AWS_AUTH_PREFIX}/include")
+    file(MAKE_DIRECTORY "${AWS_AUTH_LIB_DIR}")
+
+    find_package(OpenSSL REQUIRED)
+
+    set(AWS_IO_DEPENDS "aws-c-common_ep;aws-c-cal_ep")
+    if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        paimon_add_s2n_project()
+        list(APPEND AWS_IO_DEPENDS s2n_ep)
+    endif()
+
+    paimon_add_aws_c_project(aws-c-common ${PAIMON_AWS_C_COMMON_BUILD_VERSION}
+                             ${PAIMON_AWS_C_COMMON_BUILD_SHA256_CHECKSUM} "")
+    paimon_add_aws_c_project(aws-c-sdkutils ${PAIMON_AWS_C_SDKUTILS_BUILD_VERSION}
+                             ${PAIMON_AWS_C_SDKUTILS_BUILD_SHA256_CHECKSUM}
+                             aws-c-common_ep)
+    paimon_add_aws_c_project(aws-c-cal ${PAIMON_AWS_C_CAL_BUILD_VERSION}
+                             ${PAIMON_AWS_C_CAL_BUILD_SHA256_CHECKSUM} aws-c-common_ep)
+    paimon_add_aws_c_project(aws-c-compression ${PAIMON_AWS_C_COMPRESSION_BUILD_VERSION}
+                             ${PAIMON_AWS_C_COMPRESSION_BUILD_SHA256_CHECKSUM}
+                             aws-c-common_ep)
+    paimon_add_aws_c_project(aws-c-io ${PAIMON_AWS_C_IO_BUILD_VERSION}
+                             ${PAIMON_AWS_C_IO_BUILD_SHA256_CHECKSUM} "${AWS_IO_DEPENDS}")
+    paimon_add_aws_c_project(aws-c-http ${PAIMON_AWS_C_HTTP_BUILD_VERSION}
+                             ${PAIMON_AWS_C_HTTP_BUILD_SHA256_CHECKSUM}
+                             "aws-c-common_ep;aws-c-io_ep;aws-c-compression_ep")
+    paimon_add_aws_c_project(aws-c-auth
+                             ${PAIMON_AWS_C_AUTH_BUILD_VERSION}
+                             ${PAIMON_AWS_C_AUTH_BUILD_SHA256_CHECKSUM}
+                             "aws-c-common_ep;aws-c-sdkutils_ep;aws-c-cal_ep;aws-c-io_ep;aws-c-http_ep"
+    )
+
+    find_package(Threads REQUIRED)
+    set(AWS_AUTH_PLATFORM_LIBS Threads::Threads ${CMAKE_DL_LIBS})
+    if(APPLE)
+        list(APPEND
+             AWS_AUTH_PLATFORM_LIBS
+             "-framework Security"
+             "-framework CoreFoundation"
+             "-framework Network")
+    endif()
+    set(AWS_AUTH_TLS_LIBS)
+    if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        list(APPEND AWS_AUTH_TLS_LIBS "${AWS_AUTH_LIB_DIR}/libs2n.a")
+    endif()
+    set(AWS_AUTH_RUNTIME_LIBS
+        "${AWS_AUTH_LIB_DIR}/libaws-c-http.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-io.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-compression.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-cal.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-sdkutils.a"
+        "${AWS_AUTH_LIB_DIR}/libaws-c-common.a"
+        ${AWS_AUTH_TLS_LIBS})
+    if(NOT APPLE)
+        list(PREPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--start-group")
+        list(APPEND AWS_AUTH_RUNTIME_LIBS "-Wl,--end-group")
+    endif()
+    # Keep OpenSSL after the static AWS archives. In embedded builds CMake can
+    # resolve these targets to static archives, where link order matters.
+    list(APPEND AWS_AUTH_RUNTIME_LIBS "${OPENSSL_SSL_LIBRARY}"
+         "${OPENSSL_CRYPTO_LIBRARY}")
+    add_library(aws_auth_minimal STATIC IMPORTED GLOBAL)
+    set_target_properties(aws_auth_minimal
+                          PROPERTIES IMPORTED_LOCATION
+                                     "${AWS_AUTH_LIB_DIR}/libaws-c-auth.a"
+                                     INTERFACE_INCLUDE_DIRECTORIES
+                                     "${AWS_AUTH_PREFIX}/include"
+                                     INTERFACE_LINK_LIBRARIES
+                                     "${AWS_AUTH_RUNTIME_LIBS};${AWS_AUTH_PLATFORM_LIBS}")
+    add_dependencies(aws_auth_minimal aws-c-auth_ep)
+
+    set(AWS_AUTH_INCLUDE_DIR
+        "${AWS_AUTH_PREFIX}/include"
+        PARENT_SCOPE)
+    set(AWS_AUTH_LIB_DIR
+        "${AWS_AUTH_LIB_DIR}"
+        PARENT_SCOPE)
+endfunction()

cmake_modules/BuildUtils.cmake

@@ -180,7 +180,14 @@ function(add_paimon_lib LIB_NAME)
         target_link_libraries(${LIB_NAME}_shared
                               PUBLIC "$<BUILD_INTERFACE:paimon_sanitizer_flags>")
 
-        if(NOT APPLE)
+        if(APPLE)
+            # Arrow's checked_cast uses dynamic_cast in Debug builds, which
+            # fails across dylib boundaries on macOS due to duplicate RTTI
+            # from static Arrow archives.  Define NDEBUG so checked_cast
+            # falls back to static_cast (same as Release mode).
+            target_compile_definitions(${LIB_NAME}_objlib
+                                       PRIVATE $<$<CONFIG:Debug>:NDEBUG>)
+        else()
             set(SHARED_LINK_OPTIONS -Wl,--exclude-libs,ALL -Wl,-Bsymbolic
                                     -Wl,--gc-sections)
             # -z defs (--no-undefined) rejects the __asan_*/__tsan_*/__ubsan_* symbols that

cmake_modules/SetupCxxFlags.cmake

@@ -75,7 +75,6 @@ if("${BUILD_WARNING_LEVEL}" STREQUAL "CHECKIN")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wextra")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wdocumentation")
-        set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wglobal-constructors")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-missing-braces")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unused-parameter")
         set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-unknown-warning-option")

cmake_modules/ThirdpartyToolchain.cmake

@@ -1032,7 +1032,8 @@ macro(build_re2)
     )
     set(RE2_LIBRARIES ${RE2_STATIC_LIB})
 
-    set(RE2_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${RE2_PREFIX}")
+    set(RE2_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${RE2_PREFIX}"
+                       "-DRE2_BUILD_TESTING=OFF")
 
     externalproject_add(re2_ep
                         ${EP_COMMON_OPTIONS}
@@ -1887,6 +1888,10 @@ if(PAIMON_ENABLE_JINDO)
     build_jindosdk_c()
     build_jindosdk_nextarch()
 endif()
+if(PAIMON_ENABLE_S3)
+    include(BuildAwsAuth)
+    build_aws_auth()
+endif()
 if(PAIMON_ENABLE_LUMINA)
     build_lumina()
 endif()

cmake_modules/arrow.diff

@@ -415,6 +415,14 @@ diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/Thi
  # and crosscompiling emulator (for try_run() )
  if(CMAKE_CROSSCOMPILING_EMULATOR)
    string(REPLACE ";" ${EP_LIST_SEPARATOR} EP_CMAKE_CROSSCOMPILING_EMULATOR
+@@ -1720,6 +1725,7 @@ macro(build_thrift)
+       -DWITH_JAVASCRIPT=OFF
+       -DWITH_LIBEVENT=OFF
+       -DWITH_NODEJS=OFF
++      -DWITH_OPENSSL=OFF
+       -DWITH_PYTHON=OFF
+       -DWITH_QT5=OFF
+       -DWITH_ZLIB=OFF)
 diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake
 --- a/cpp/cmake_modules/BuildUtils.cmake
 +++ b/cpp/cmake_modules/BuildUtils.cmake

src/paimon/CMakeLists.txt

@@ -177,6 +177,12 @@ set(PAIMON_COMMON_SRCS
     common/utils/status.cpp
     common/utils/string_utils.cpp)
 
+if(PAIMON_ENABLE_S3)
+    list(APPEND PAIMON_COMMON_SRCS common/fs/http_client.cpp
+         common/fs/object_store_file_system.cpp)
+    set(PAIMON_OBJECT_STORE_LINK_LIBS CURL::libcurl)
+endif()
+
 set(PAIMON_CORE_SRCS
     core/disk/file_io_channel.cpp
     core/mergetree/spill_reader.cpp
@@ -409,6 +415,7 @@ add_paimon_lib(paimon
                xxhash
                Threads::Threads
                RapidJSON
+               ${PAIMON_OBJECT_STORE_LINK_LIBS}
                STATIC_LINK_LIBS
                arrow
                tbb
@@ -418,6 +425,7 @@ add_paimon_lib(paimon
                xxhash
                Threads::Threads
                RapidJSON
+               ${PAIMON_OBJECT_STORE_LINK_LIBS}
                SHARED_LINK_FLAGS
                ${PAIMON_VERSION_SCRIPT_FLAGS})
 
@@ -837,10 +845,17 @@ if(PAIMON_BUILD_TESTS)
              ${PAIMON_JINDO_FILE_SYSTEM_STATIC_LINK_LIBS})
     endif()
 
+    set(PAIMON_OBJECT_STORE_FS_TEST_SOURCES)
+    if(PAIMON_ENABLE_S3)
+        list(APPEND PAIMON_OBJECT_STORE_FS_TEST_SOURCES
+             common/fs/object_store_file_system_test.cpp)
+    endif()
+
     add_paimon_test(fs_test
                     SOURCES
                     common/fs/file_system_test.cpp
                     common/fs/resolving_file_system_test.cpp
+                    ${PAIMON_OBJECT_STORE_FS_TEST_SOURCES}
                     fs/local/local_file_test.cpp
                     ${PAIMON_JINDO_FS_TEST_SOURCES}
                     STATIC_LINK_LIBS