diff --git a/.github/workflows/cross-compile.yml b/.github/workflows/cross-compile.yml new file mode 100644 index 00000000..d2c7111d --- /dev/null +++ b/.github/workflows/cross-compile.yml @@ -0,0 +1,177 @@ +name: Cross compile + +on: + push: + branches: [main] + pull_request: + branches: [main] + workflow_dispatch: {} + +env: + UBUNTU_SNAPSHOT: 20260825T200000Z + +jobs: + build: + name: ${{ matrix.host_arch }} to ${{ matrix.target_arch }} + strategy: + fail-fast: false + matrix: + include: + - host_arch: amd64 + target_arch: arm64 + runner: ubuntu-24.04 + compiler_package: gcc-aarch64-linux-gnu=4:13.2.0-7ubuntu1 + cxx_compiler_package: g++-aarch64-linux-gnu=4:13.2.0-7ubuntu1 + toolchain: CMake/toolchains/linux-aarch64-gnu.cmake + arch_flags: -march=armv8.2-a -mtune=generic + file_pattern: ARM aarch64 + - host_arch: arm64 + target_arch: amd64 + runner: ubuntu-24.04-arm + compiler_package: gcc-x86-64-linux-gnu=4:13.2.0-7ubuntu1 + cxx_compiler_package: g++-x86-64-linux-gnu=4:13.2.0-7ubuntu1 + toolchain: CMake/toolchains/linux-x86_64-gnu.cmake + arch_flags: -march=x86-64-v3 -mtune=generic + file_pattern: x86-64 + runs-on: ${{ matrix.runner }} + permissions: + contents: read + + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + + - name: Configure pinned Ubuntu package snapshot + env: + TARGET_ARCH: ${{ matrix.target_arch }} + run: | + sudo dpkg --add-architecture "$TARGET_ARCH" + sudo rm -f /etc/apt/sources.list + sudo rm -f /etc/apt/sources.list.d/* + cat <> "$GITHUB_OUTPUT" + + - name: Prepare apt archive cache + run: | + sudo rm -f /etc/apt/apt.conf.d/docker-clean + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' | sudo tee /etc/apt/apt.conf.d/keep-cache + sudo chmod 0777 /var/cache/apt/archives + + - name: Cache apt archives + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: /var/cache/apt/archives + key: apt-noble-${{ env.UBUNTU_SNAPSHOT }}-${{ matrix.host_arch }}-${{ matrix.target_arch }}-${{ hashFiles('.github/workflows/cross-*-packages.lock') }} + + - name: Install pinned cross-build packages + env: + PACKAGES: ${{ steps.packages.outputs.packages }} + run: | + # Word splitting is intentional: PACKAGES contains apt package arguments. + # shellcheck disable=SC2086 + sudo apt-get install -y --allow-downgrades --no-install-recommends $PACKAGES + + - name: Configure + env: + ARCH_FLAGS: ${{ matrix.arch_flags }} + run: | + cmake -S . -B build -G "Unix Makefiles" \ + -DCMAKE_TOOLCHAIN_FILE="${{ matrix.toolchain }}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="$ARCH_FLAGS" \ + -DCMAKE_CXX_FLAGS="$ARCH_FLAGS" \ + -DBUILD_TESTING=OFF \ + -DENABLE_DSA=OFF \ + -DENABLE_ISAL=OFF + + - name: Build at maximum parallelism + run: cmake --build build --parallel "$(nproc)" + + - name: Verify target architecture + env: + FILE_PATTERN: ${{ matrix.file_pattern }} + run: | + file build/output/overlaybd-tcmu + file build/output/overlaybd-tcmu | grep -F "$FILE_PATTERN" + + optional-dependencies: + name: Optional dependency (${{ matrix.feature }}) + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + include: + - feature: bundled-curl + cmake_flags: -DBUILD_CURL_FROM_SOURCE=ON + target: overlaybd-tcmu + - feature: dsa + cmake_flags: -DENABLE_DSA=ON + target: zfile_lib + - feature: isa-l + cmake_flags: -DENABLE_ISAL=ON + target: zfile_lib + permissions: + contents: read + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + submodules: recursive + + - name: Configure pinned Ubuntu snapshot and apt cache + run: | + sudo rm -f /etc/apt/sources.list + sudo rm -f /etc/apt/sources.list.d/* + cat < + #include ++#include + #include diff --git a/CMake/patches/photon-v0.6.17-ext4-rootfs-format.patch b/CMake/patches/photon-v0.6.17-ext4-rootfs-format.patch index e7b950f9..005ab4cc 100644 --- a/CMake/patches/photon-v0.6.17-ext4-rootfs-format.patch +++ b/CMake/patches/photon-v0.6.17-ext4-rootfs-format.patch @@ -87,3 +87,19 @@ index 0195ae5..60a0aa7 100644 } ret = ext2fs_close_free(&fs); +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 9a9d71c..0638643 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -68,5 +68,8 @@ +-if (${ARCH} STREQUAL x86_64) ++if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64)$") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2") +-elseif (${ARCH} STREQUAL aarch64) +- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=generic+crc -fsigned-char -fno-stack-protector -fomit-frame-pointer") ++elseif (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") ++ if (NOT CMAKE_CXX_FLAGS MATCHES "(^|[ ;])-m(arch|cpu)=") ++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcpu=generic+crc") ++ endif() ++ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsigned-char -fno-stack-protector -fomit-frame-pointer") + endif () diff --git a/CMake/toolchains/linux-aarch64-gnu.cmake b/CMake/toolchains/linux-aarch64-gnu.cmake new file mode 100644 index 00000000..2a799f17 --- /dev/null +++ b/CMake/toolchains/linux-aarch64-gnu.cmake @@ -0,0 +1,20 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR aarch64) +set(OVERLAYBD_TARGET_TRIPLET aarch64-linux-gnu CACHE STRING "GNU target triplet") + +set(CMAKE_C_COMPILER aarch64-linux-gnu-gcc) +set(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++) +set(CMAKE_AR aarch64-linux-gnu-ar) +set(CMAKE_RANLIB aarch64-linux-gnu-ranlib) +set(CMAKE_STRIP aarch64-linux-gnu-strip) + +set(CMAKE_LIBRARY_ARCHITECTURE aarch64-linux-gnu) +set(CMAKE_FIND_ROOT_PATH /usr/aarch64-linux-gnu /usr) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +set(ENV{PKG_CONFIG_LIBDIR} + "/usr/lib/aarch64-linux-gnu/pkgconfig:/usr/share/pkgconfig") +set(ENV{PKG_CONFIG_PATH} "") diff --git a/CMake/toolchains/linux-x86_64-gnu.cmake b/CMake/toolchains/linux-x86_64-gnu.cmake new file mode 100644 index 00000000..fe8603f7 --- /dev/null +++ b/CMake/toolchains/linux-x86_64-gnu.cmake @@ -0,0 +1,20 @@ +set(CMAKE_SYSTEM_NAME Linux) +set(CMAKE_SYSTEM_PROCESSOR x86_64) +set(OVERLAYBD_TARGET_TRIPLET x86_64-linux-gnu CACHE STRING "GNU target triplet") + +set(CMAKE_C_COMPILER x86_64-linux-gnu-gcc) +set(CMAKE_CXX_COMPILER x86_64-linux-gnu-g++) +set(CMAKE_AR x86_64-linux-gnu-ar) +set(CMAKE_RANLIB x86_64-linux-gnu-ranlib) +set(CMAKE_STRIP x86_64-linux-gnu-strip) + +set(CMAKE_LIBRARY_ARCHITECTURE x86_64-linux-gnu) +set(CMAKE_FIND_ROOT_PATH /usr/x86_64-linux-gnu /usr) +set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) +set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) +set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + +set(ENV{PKG_CONFIG_LIBDIR} + "/usr/lib/x86_64-linux-gnu/pkgconfig:/usr/share/pkgconfig") +set(ENV{PKG_CONFIG_PATH} "") diff --git a/CMakeLists.txt b/CMakeLists.txt index 5ad53464..e4cbf096 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,47 +6,69 @@ project( ) enable_language(C) +include(CheckCXXCompilerFlag) + +set(OVERLAYBD_SUBBUILD_JOBS 1 CACHE STRING + "Parallel jobs for nested non-CMake dependency builds") +if(OVERLAYBD_SUBBUILD_JOBS LESS 1) + message(FATAL_ERROR "OVERLAYBD_SUBBUILD_JOBS must be at least 1") +endif() +find_program(OVERLAYBD_MAKE_EXECUTABLE NAMES gmake make) +if(NOT OVERLAYBD_MAKE_EXECUTABLE) + message(FATAL_ERROR "A make implementation is required for bundled dependencies") +endif() + set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "-O2 -DNDEBUG -g") -# Get CPU arch -execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE) -if (NOT (${ARCH} STREQUAL x86_64) AND NOT (${ARCH} STREQUAL aarch64) AND NOT (${ARCH} STREQUAL arm64)) - message(FATAL_ERROR "Unknown CPU architecture ${ARCH}") +string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" OVERLAYBD_SYSTEM_PROCESSOR) +if(NOT OVERLAYBD_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|aarch64|arm64)$") + message(FATAL_ERROR "Unknown CPU architecture ${CMAKE_SYSTEM_PROCESSOR}") endif () -option(OBD_VER "Overlaybd version" "overlaybd/0.0.0-undefined") +set(OBD_VER "overlaybd/0.0.0-undefined" CACHE STRING "Overlaybd version") -set(LIBRARY_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/build/output") -set(EXECUTABLE_OUTPUT_PATH "${CMAKE_SOURCE_DIR}/build/output") +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/output") +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/output") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/output") +# Retain these aliases for the existing test and packaging CMake files. +set(LIBRARY_OUTPUT_PATH "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}") +set(EXECUTABLE_OUTPUT_PATH "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}") set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/CMake") -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fpic -Wall -Werror=sign-compare") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fpic -Wall -Werror=sign-compare -DOVERLAYBD_VER=${OBD_VER}") - -if (${ARCH} STREQUAL aarch64) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc -fsigned-char -fno-stack-protector -fomit-frame-pointer") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +add_compile_options(-Wall -Werror=sign-compare) +add_compile_definitions(OVERLAYBD_VER=${OBD_VER}) + +if(OVERLAYBD_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") + set(_overlaybd_arch_flags "${CMAKE_C_FLAGS} ${CMAKE_CXX_FLAGS}") + if(NOT _overlaybd_arch_flags MATCHES "(^|[ ;])-m(arch|cpu)=") + add_compile_options(-march=armv8-a+crc) + endif() + add_compile_options($<$:-fsigned-char> + $<$:-fno-stack-protector> + $<$:-fomit-frame-pointer>) endif () set(CMAKE_CXX_STANDARD_LIBRARIES "-static-libgcc ${CMAKE_CXX_STANDARD_LIBRARIES}") -find_library(STATIC_LIBSTDC++ libstdc++.a PATHS /usr/lib/gcc/*/*) -if(NOT ${STATIC_LIBSTDC++} STREQUAL "STATIC_LIBSTDC++-NOTFOUND") +find_library(STATIC_LIBSTDCXX libstdc++.a) +if(STATIC_LIBSTDCXX) set(CMAKE_CXX_STANDARD_LIBRARIES "-static-libstdc++ ${CMAKE_CXX_STANDARD_LIBRARIES}") endif() -find_library(LIBZSTD libzstd.a PATHS /usr/lib/x86_64-linux-gnu/* /usr/lib64/*) -if(${LIBZSTD} STREQUAL "LIBZSTD-NOTFOUND") - message("libzstd.a not found, try to find shared library") - find_library(LIBZSTD libzstd.so PATHS /usr/lib/x86_64-linux-gnu/* /usr/lib64/*) -endif() - set(CMAKE_CXX_STANDARD 14) set(CMAKE_CXX_STANDARD_REQUIRED on) set(ENABLE_MIMIC_VDSO off) option(BUILD_CURL_FROM_SOURCE "Compile static libcurl" off) option(ORIGIN_EXT2FS "Use original libext2fs" off) +option(ENABLE_DSA "Build the Intel DML crc32 backend" off) +option(ENABLE_ISAL "Build the Intel ISA-L crc32 backend" off) +if((ENABLE_DSA OR ENABLE_ISAL) AND NOT OVERLAYBD_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64)$") + message(FATAL_ERROR "ENABLE_DSA and ENABLE_ISAL are supported only on x86_64") +endif() find_package(photon REQUIRED) find_package(tcmu REQUIRED) find_package(yamlcpp) diff --git a/baselayers/CMakeLists.txt b/baselayers/CMakeLists.txt index 7e7d7f54..5f53eff6 100644 --- a/baselayers/CMakeLists.txt +++ b/baselayers/CMakeLists.txt @@ -1,9 +1,11 @@ add_custom_command( OUTPUT ${EXECUTABLE_OUTPUT_PATH}/ext4_64 + COMMAND ${CMAKE_COMMAND} -E make_directory ${EXECUTABLE_OUTPUT_PATH} COMMAND tar -zxvf ${CMAKE_CURRENT_LIST_DIR}/ext4_64.tar.gz -C ${EXECUTABLE_OUTPUT_PATH} + DEPENDS ${CMAKE_CURRENT_LIST_DIR}/ext4_64.tar.gz VERBATIM ) add_custom_target(baselayer ALL DEPENDS ${EXECUTABLE_OUTPUT_PATH}/ext4_64) -install(FILES ${EXECUTABLE_OUTPUT_PATH}/ext4_64 DESTINATION /opt/overlaybd/baselayers) \ No newline at end of file +install(FILES ${EXECUTABLE_OUTPUT_PATH}/ext4_64 DESTINATION /opt/overlaybd/baselayers) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4d0e696e..5604d1a9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -26,10 +26,10 @@ target_include_directories(overlaybd_image_lib PUBLIC target_link_libraries(overlaybd_image_lib photon_static overlaybd_lib - ${CURL_LIBRARIES} - ${OPENSSL_SSL_LIBRARY} - ${OPENSSL_CRYPTO_LIBRARY} - ${AIO_LIBRARIES} + CURL::libcurl + OpenSSL::SSL + OpenSSL::Crypto + AIO::aio ) add_executable(overlaybd-tcmu @@ -47,10 +47,10 @@ target_link_libraries(overlaybd-tcmu overlaybd_lib overlaybd_image_lib tcmu_static - ${CURL_LIBRARIES} - ${OPENSSL_SSL_LIBRARY} - ${OPENSSL_CRYPTO_LIBRARY} - ${AIO_LIBRARIES} + CURL::libcurl + OpenSSL::SSL + OpenSSL::Crypto + AIO::aio ) install(TARGETS overlaybd-tcmu DESTINATION /opt/overlaybd/bin) diff --git a/src/overlaybd/CMakeLists.txt b/src/overlaybd/CMakeLists.txt index ff1646ae..f78e89d8 100644 --- a/src/overlaybd/CMakeLists.txt +++ b/src/overlaybd/CMakeLists.txt @@ -1,13 +1,13 @@ +add_subdirectory(otel) add_subdirectory(registryfs) add_subdirectory(lsmt) -add_subdirectory(zfile) add_subdirectory(zstd) +add_subdirectory(zfile) add_subdirectory(cache) add_subdirectory(tar) add_subdirectory(gzip) add_subdirectory(gzindex) add_subdirectory(stream_convertor) -add_subdirectory(otel) add_library(overlaybd_lib INTERFACE) target_include_directories(overlaybd_lib INTERFACE diff --git a/src/overlaybd/cache/ocf_cache/test/CMakeLists.txt b/src/overlaybd/cache/ocf_cache/test/CMakeLists.txt index 6c629cc1..cb9302b9 100644 --- a/src/overlaybd/cache/ocf_cache/test/CMakeLists.txt +++ b/src/overlaybd/cache/ocf_cache/test/CMakeLists.txt @@ -11,7 +11,7 @@ target_include_directories( ) target_link_libraries( ocf_perf_test - gflags pthread ${CURL_LIBRARIES} + gflags pthread CURL::libcurl photon_static overlaybd_lib ) diff --git a/src/overlaybd/otel/CMakeLists.txt b/src/overlaybd/otel/CMakeLists.txt index f4021b7f..72fa2f35 100644 --- a/src/overlaybd/otel/CMakeLists.txt +++ b/src/overlaybd/otel/CMakeLists.txt @@ -8,18 +8,41 @@ FetchContent_Declare( opentelemetry-cpp GIT_REPOSITORY https://github.com/open-telemetry/opentelemetry-cpp.git GIT_TAG v1.22.0 + GIT_SHALLOW TRUE + GIT_SUBMODULES + third_party/nlohmann-json + third_party/opentelemetry-proto ) -# Set OpenTelemetry build options before making it available +# Set OpenTelemetry build options without leaking them into OverlayBD's cache. +set(_otel_saved_BUILD_TESTING "${BUILD_TESTING}") +set(_otel_saved_WITH_EXAMPLES "${WITH_EXAMPLES}") +set(_otel_saved_OPENTELEMETRY_INSTALL "${OPENTELEMETRY_INSTALL}") +set(_otel_saved_WITH_OTLP_GRPC "${WITH_OTLP_GRPC}") +set(_otel_saved_WITH_OTLP_HTTP "${WITH_OTLP_HTTP}") +set(_otel_saved_WITH_FUNC_TESTS "${WITH_FUNC_TESTS}") +set(_otel_saved_BUILD_SHARED_LIBS "${BUILD_SHARED_LIBS}") set(BUILD_TESTING OFF CACHE BOOL "Build tests" FORCE) set(WITH_EXAMPLES OFF CACHE BOOL "Build examples" FORCE) set(OPENTELEMETRY_INSTALL ON CACHE BOOL "Install OpenTelemetry" FORCE) set(WITH_OTLP_GRPC OFF CACHE BOOL "Build with OTLP GRPC" FORCE) set(WITH_OTLP_HTTP ON CACHE BOOL "Build with OTLP HTTP" FORCE) +set(WITH_FUNC_TESTS OFF CACHE BOOL "Build functional tests" FORCE) set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build with shared library" FORCE) FetchContent_MakeAvailable(opentelemetry-cpp) +set(BUILD_TESTING "${_otel_saved_BUILD_TESTING}" CACHE BOOL "Build tests" FORCE) +set(WITH_EXAMPLES "${_otel_saved_WITH_EXAMPLES}" CACHE BOOL "Build examples" FORCE) +set(OPENTELEMETRY_INSTALL "${_otel_saved_OPENTELEMETRY_INSTALL}" CACHE BOOL + "Install OpenTelemetry" FORCE) +set(WITH_OTLP_GRPC "${_otel_saved_WITH_OTLP_GRPC}" CACHE BOOL "Build OTLP gRPC" FORCE) +set(WITH_OTLP_HTTP "${_otel_saved_WITH_OTLP_HTTP}" CACHE BOOL "Build OTLP HTTP" FORCE) +set(WITH_FUNC_TESTS "${_otel_saved_WITH_FUNC_TESTS}" CACHE BOOL + "Build functional tests" FORCE) +set(BUILD_SHARED_LIBS "${_otel_saved_BUILD_SHARED_LIBS}" CACHE BOOL + "Build shared libraries" FORCE) + add_library(otel_lib STATIC ${SOURCE_OTEL}) set_target_properties(otel_lib PROPERTIES LINKER_LANGUAGE CXX) diff --git a/src/overlaybd/tar/erofs/CMakeLists.txt b/src/overlaybd/tar/erofs/CMakeLists.txt index 6027102e..37187912 100644 --- a/src/overlaybd/tar/erofs/CMakeLists.txt +++ b/src/overlaybd/tar/erofs/CMakeLists.txt @@ -6,24 +6,80 @@ FetchContent_Declare( GIT_TAG eec6f7a2755dfccc8f655aa37cf6f26db9164e60 ) -FetchContent_MakeAvailable(erofs-utils) +FetchContent_GetProperties(erofs-utils) +if(NOT erofs-utils_POPULATED) + FetchContent_Populate(erofs-utils) +endif() -execute_process( - COMMAND ./autogen.sh - WORKING_DIRECTORY ${erofs-utils_SOURCE_DIR} -) -execute_process( - COMMAND ./configure --disable-lz4 --disable-lzma --without-libzstd --without-uuid --disable-multithreading - WORKING_DIRECTORY ${erofs-utils_SOURCE_DIR} -) -execute_process( - COMMAND make - WORKING_DIRECTORY ${erofs-utils_SOURCE_DIR} -) +set(EROFS_BUILD_SOURCE_DIR "${erofs-utils_BINARY_DIR}/source") +set(EROFS_LIB_INCLUDE_DIR "${EROFS_BUILD_SOURCE_DIR}/include/" CACHE PATH + "erofs-utils include path") +set(EROFS_CONFIG_FILE "${EROFS_BUILD_SOURCE_DIR}/config.h" CACHE FILEPATH + "erofs-utils config file") +set(EROFS_LIB_STATIC "${EROFS_BUILD_SOURCE_DIR}/lib/.libs/liberofs.a" CACHE FILEPATH + "erofs-utils static library") +set(_erofs_build_signature "${erofs-utils_BINARY_DIR}/overlaybd-build-config") +file(GENERATE OUTPUT "${_erofs_build_signature}" CONTENT + "CC=${CMAKE_C_COMPILER}\nAR=${CMAKE_AR}\nRANLIB=${CMAKE_RANLIB}\nCFLAGS=${CMAKE_C_FLAGS}\nTRIPLET=${OVERLAYBD_TARGET_TRIPLET}\n") + +set(_erofs_configure_args + --disable-lz4 + --disable-lzma + --without-libzstd + --without-uuid + --disable-multithreading) +if(CMAKE_CROSSCOMPILING) + if(OVERLAYBD_TARGET_TRIPLET) + list(APPEND _erofs_configure_args "--host=${OVERLAYBD_TARGET_TRIPLET}") + elseif(CMAKE_C_COMPILER_TARGET) + list(APPEND _erofs_configure_args "--host=${CMAKE_C_COMPILER_TARGET}") + elseif(OVERLAYBD_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") + list(APPEND _erofs_configure_args --host=aarch64-linux-gnu) + else() + list(APPEND _erofs_configure_args --host=x86_64-linux-gnu) + endif() +endif() -set(EROFS_LIB_INCLUDE_DIR "${erofs-utils_SOURCE_DIR}/include/" CACHE PATH "erofs-utils include path.") -set(EROFS_CONFIG_FILE "${erofs-utils_SOURCE_DIR}/config.h" CACHE PATH "erofs-utils config file.") -set(EROFS_LIB_STATIC "${erofs-utils_SOURCE_DIR}/lib/.libs/liberofs.a" CACHE PATH "erofs-utils static lib.") +add_custom_command( + OUTPUT "${EROFS_LIB_STATIC}" "${EROFS_CONFIG_FILE}" + COMMAND ${CMAKE_COMMAND} -E remove_directory "${EROFS_BUILD_SOURCE_DIR}" + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${erofs-utils_SOURCE_DIR}" "${EROFS_BUILD_SOURCE_DIR}" + COMMAND ${CMAKE_COMMAND} -E chdir "${EROFS_BUILD_SOURCE_DIR}" + ${CMAKE_COMMAND} -E env + "CC=${CMAKE_C_COMPILER}" + "AR=${CMAKE_AR}" + "RANLIB=${CMAKE_RANLIB}" + "CFLAGS=${CMAKE_C_FLAGS} -fPIC" + /bin/sh ./autogen.sh + COMMAND ${CMAKE_COMMAND} -E chdir "${EROFS_BUILD_SOURCE_DIR}" + ${CMAKE_COMMAND} -E env + "CC=${CMAKE_C_COMPILER}" + "AR=${CMAKE_AR}" + "RANLIB=${CMAKE_RANLIB}" + "CFLAGS=${CMAKE_C_FLAGS} -fPIC" + /bin/sh ./configure ${_erofs_configure_args} + COMMAND ${CMAKE_COMMAND} -E chdir "${EROFS_BUILD_SOURCE_DIR}" + ${OVERLAYBD_MAKE_EXECUTABLE} -j${OVERLAYBD_SUBBUILD_JOBS} + WORKING_DIRECTORY "${erofs-utils_BINARY_DIR}" + DEPENDS + "${erofs-utils_SOURCE_DIR}/autogen.sh" + "${erofs-utils_SOURCE_DIR}/configure.ac" + "${erofs-utils_SOURCE_DIR}/Makefile.am" + "${_erofs_build_signature}" + "${CMAKE_CURRENT_LIST_FILE}" + VERBATIM) +add_custom_target(erofs_utils_build DEPENDS "${EROFS_LIB_STATIC}" "${EROFS_CONFIG_FILE}") + +file(MAKE_DIRECTORY "${EROFS_LIB_INCLUDE_DIR}") +if(NOT TARGET EROFS::utils) + add_library(EROFS::utils STATIC IMPORTED GLOBAL) + set_target_properties(EROFS::utils PROPERTIES + IMPORTED_LOCATION "${EROFS_LIB_STATIC}" + INTERFACE_INCLUDE_DIRECTORIES "${EROFS_LIB_INCLUDE_DIR}" + INTERFACE_COMPILE_OPTIONS "-include${EROFS_CONFIG_FILE}") + add_dependencies(EROFS::utils erofs_utils_build) +endif() file(GLOB EROFS_SOURCE "*.cpp") @@ -33,12 +89,7 @@ target_include_directories(erofs_lib PRIVATE ${PHOTON_INCLUDE_DIR} ) -target_include_directories(erofs_lib PRIVATE - ${EROFS_LIB_INCLUDE_DIR} -) - -target_compile_options(erofs_lib PRIVATE "-include${EROFS_CONFIG_FILE}") -target_link_libraries(erofs_lib PRIVATE ${EROFS_LIB_STATIC}) +target_link_libraries(erofs_lib PRIVATE EROFS::utils) if(BUILD_TESTING) add_subdirectory(test) diff --git a/src/overlaybd/zfile/CMakeLists.txt b/src/overlaybd/zfile/CMakeLists.txt index 49af3e11..0ff2c67a 100644 --- a/src/overlaybd/zfile/CMakeLists.txt +++ b/src/overlaybd/zfile/CMakeLists.txt @@ -2,43 +2,41 @@ file(GLOB SOURCE_ZFILE "*.cpp") file(GLOB SOURCE_LZ4 "lz4/*.c") file(GLOB SOURCE_CRC32 "crc32/crc32c.cpp") -# Get CPU arch -execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH OUTPUT_STRIP_TRAILING_WHITESPACE) -if (NOT (${ARCH} STREQUAL x86_64) AND NOT (${ARCH} STREQUAL aarch64)) - message(FATAL_ERROR "Unknown CPU architecture ${ARCH}") -endif () - set (CMAKE_CXX_STANDARD 17) add_library(crc32_lib STATIC ${SOURCE_CRC32}) target_include_directories(crc32_lib PUBLIC ${PHOTON_INCLUDE_DIR} ) -if (${ARCH} STREQUAL x86_64) +if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64) target_compile_options(crc32_lib PUBLIC -msse4.2 -mcrc32) else() - target_compile_options(crc32_lib PRIVATE -march=native -mcpu=generic+crc) + if (NOT CMAKE_CXX_FLAGS MATCHES "-march=|-mcpu=") + check_cxx_compiler_flag(-mcpu=native COMPILER_HAS_NATIVE_FLAG) + if (COMPILER_HAS_NATIVE_FLAG) + target_compile_options(crc32_lib PRIVATE -mcpu=native) + else () + target_compile_options(crc32_lib PRIVATE -mcpu=generic+crc) + endif () + endif () endif() if(ENABLE_DSA OR ENABLE_ISAL) add_subdirectory(thirdparty) - add_dependencies(crc32_lib thirdparty_lib) - target_link_directories(crc32_lib PUBLIC ${LIBRARY_OUTPUT_PATH}) - target_include_directories(crc32_lib PUBLIC ${LIBRARY_OUTPUT_PATH}/include) if(ENABLE_DSA) - target_link_libraries(crc32_lib -lpci -ldmlhl -ldl) + target_link_libraries(crc32_lib PUBLIC DML::dmlhl pci dl) target_compile_definitions(crc32_lib PUBLIC -DENABLE_DSA) endif() if(ENABLE_ISAL) target_compile_options(crc32_lib PUBLIC -mavx512f) target_compile_definitions(crc32_lib PUBLIC -DENABLE_ISAL) - target_link_libraries(crc32_lib -lisal) + target_link_libraries(crc32_lib PUBLIC ISA_L::isal) endif() endif() set (CMAKE_CXX_STANDARD 14) add_library(zfile_lib STATIC ${SOURCE_ZFILE} ${SOURCE_LZ4}) -target_link_libraries(zfile_lib photon_static crc32_lib ${LIBZSTD}) +target_link_libraries(zfile_lib photon_static crc32_lib libzstd_static) if (ENABLE_QAT) target_compile_definitions(zfile_lib PUBLIC -DENABLE_QAT) diff --git a/src/overlaybd/zfile/thirdparty/CMakeLists.txt b/src/overlaybd/zfile/thirdparty/CMakeLists.txt index 4dec6f2a..46888dd3 100644 --- a/src/overlaybd/zfile/thirdparty/CMakeLists.txt +++ b/src/overlaybd/zfile/thirdparty/CMakeLists.txt @@ -1,23 +1,101 @@ -add_custom_target(thirdparty_lib) -set(THIRDPARTY_PATH "${CMAKE_SOURCE_DIR}/src/overlaybd/zfile/thirdparty") +include(ExternalProject) if(ENABLE_DSA) - add_custom_command(TARGET thirdparty_lib - COMMAND cd ${THIRDPARTY_PATH}/DML && git checkout 5a2956362d7b1c65d8aee753938ae9bf5cf0b7fd - COMMAND cd ${THIRDPARTY_PATH}/DML && rm -rf build && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=dmldir .. - COMMAND cd ${THIRDPARTY_PATH}/DML/build && cmake --build . --target install - COMMAND cd ${THIRDPARTY_PATH}/DML/build/dmldir && find ./ -name "libdml*.a" | xargs -i cp {} ${LIBRARY_OUTPUT_PATH}/; - COMMAND cp -r ${THIRDPARTY_PATH}/DML/build/dmldir/include ${LIBRARY_OUTPUT_PATH}/ - ) + set(DML_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/dml-install") + set(DML_LIBRARY "${DML_INSTALL_DIR}/lib/libdmlhl.a") + set(_dml_utility_patch "${CMAKE_SOURCE_DIR}/CMake/patches/dml-include-utility.patch") + file(MAKE_DIRECTORY "${DML_INSTALL_DIR}/include") + + set(_dml_cmake_args + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_INSTALL_PREFIX=${DML_INSTALL_DIR} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}) + if(CMAKE_TOOLCHAIN_FILE) + list(APPEND _dml_cmake_args -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}) + endif() + + ExternalProject_Add(dml_external + SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/DML" + BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/dml-build" + CONFIGURE_COMMAND "" + BUILD_COMMAND ${CMAKE_COMMAND} --build --parallel ${OVERLAYBD_SUBBUILD_JOBS} + INSTALL_COMMAND ${CMAKE_COMMAND} --install + BUILD_BYPRODUCTS "${DML_LIBRARY}" + PATCH_COMMAND sh -c "git apply --reverse --check '${_dml_utility_patch}' 2>/dev/null || git apply '${_dml_utility_patch}'" + UPDATE_COMMAND "") + ExternalProject_Add_Step(dml_external overlaybd_configure + COMMAND ${CMAKE_COMMAND} + -S + -B + ${_dml_cmake_args} + DEPENDEES download + DEPENDERS build + ALWAYS TRUE) + + add_library(DML::dmlhl STATIC IMPORTED GLOBAL) + set_target_properties(DML::dmlhl PROPERTIES + IMPORTED_LOCATION "${DML_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${DML_INSTALL_DIR}/include") + add_dependencies(DML::dmlhl dml_external) endif() if(ENABLE_ISAL) - add_custom_command(TARGET thirdparty_lib - COMMAND cd ${THIRDPARTY_PATH}/isa-l && git checkout ad8dce15c6d3f0c7f3d1b486d9c649ed39223b45 - COMMAND cd ${THIRDPARTY_PATH}/isa-l && ./autogen.sh - COMMAND cd ${THIRDPARTY_PATH}/isa-l && ./configure - COMMAND cd ${THIRDPARTY_PATH}/isa-l && make - COMMAND cp -r ${THIRDPARTY_PATH}/isa-l/.libs/libisal.a ${LIBRARY_OUTPUT_PATH}/ - COMMAND mkdir -p ${LIBRARY_OUTPUT_PATH}/include && cp ${THIRDPARTY_PATH}/isa-l/include/crc.h ${LIBRARY_OUTPUT_PATH}/include/ - ) -endif() \ No newline at end of file + set(ISAL_BUILD_SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/isa-l-source") + set(ISAL_LIBRARY "${ISAL_BUILD_SOURCE_DIR}/.libs/libisal.a") + set(ISAL_INCLUDE_DIR "${ISAL_BUILD_SOURCE_DIR}/include") + set(_isal_build_signature "${CMAKE_CURRENT_BINARY_DIR}/isa-l-build-config") + file(GENERATE OUTPUT "${_isal_build_signature}" CONTENT + "CC=${CMAKE_C_COMPILER}\nAR=${CMAKE_AR}\nRANLIB=${CMAKE_RANLIB}\nCFLAGS=${CMAKE_C_FLAGS}\nTRIPLET=${OVERLAYBD_TARGET_TRIPLET}\n") + file(MAKE_DIRECTORY "${ISAL_INCLUDE_DIR}") + set(_isal_configure_args --enable-static --disable-shared) + if(CMAKE_CROSSCOMPILING) + if(OVERLAYBD_TARGET_TRIPLET) + list(APPEND _isal_configure_args --host=${OVERLAYBD_TARGET_TRIPLET}) + elseif(CMAKE_C_COMPILER_TARGET) + list(APPEND _isal_configure_args --host=${CMAKE_C_COMPILER_TARGET}) + elseif(OVERLAYBD_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64)$") + list(APPEND _isal_configure_args --host=aarch64-linux-gnu) + else() + list(APPEND _isal_configure_args --host=x86_64-linux-gnu) + endif() + endif() + + add_custom_command( + OUTPUT "${ISAL_LIBRARY}" "${ISAL_INCLUDE_DIR}/crc.h" + COMMAND ${CMAKE_COMMAND} -E remove_directory "${ISAL_BUILD_SOURCE_DIR}" + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${CMAKE_CURRENT_SOURCE_DIR}/isa-l" "${ISAL_BUILD_SOURCE_DIR}" + COMMAND ${CMAKE_COMMAND} -E chdir "${ISAL_BUILD_SOURCE_DIR}" + ${CMAKE_COMMAND} -E env + "CC=${CMAKE_C_COMPILER}" + "AR=${CMAKE_AR}" + "RANLIB=${CMAKE_RANLIB}" + "CFLAGS=${CMAKE_C_FLAGS} -fPIC" + /bin/sh ./autogen.sh + COMMAND ${CMAKE_COMMAND} -E chdir "${ISAL_BUILD_SOURCE_DIR}" + ${CMAKE_COMMAND} -E env + "CC=${CMAKE_C_COMPILER}" + "AR=${CMAKE_AR}" + "RANLIB=${CMAKE_RANLIB}" + "CFLAGS=${CMAKE_C_FLAGS} -fPIC" + /bin/sh ./configure ${_isal_configure_args} + COMMAND ${CMAKE_COMMAND} -E chdir "${ISAL_BUILD_SOURCE_DIR}" + ${OVERLAYBD_MAKE_EXECUTABLE} -j${OVERLAYBD_SUBBUILD_JOBS} libisal.la + WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}" + DEPENDS + "${CMAKE_CURRENT_SOURCE_DIR}/isa-l/autogen.sh" + "${CMAKE_CURRENT_SOURCE_DIR}/isa-l/configure.ac" + "${_isal_build_signature}" + "${CMAKE_CURRENT_LIST_FILE}" + VERBATIM) + add_custom_target(isal_external DEPENDS "${ISAL_LIBRARY}" "${ISAL_INCLUDE_DIR}/crc.h") + + add_library(ISA_L::isal STATIC IMPORTED GLOBAL) + set_target_properties(ISA_L::isal PROPERTIES + IMPORTED_LOCATION "${ISAL_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${ISAL_INCLUDE_DIR}") + add_dependencies(ISA_L::isal isal_external) +endif() diff --git a/src/overlaybd/zstd/CMakeLists.txt b/src/overlaybd/zstd/CMakeLists.txt index 5c10e7ab..d2ef921f 100644 --- a/src/overlaybd/zstd/CMakeLists.txt +++ b/src/overlaybd/zstd/CMakeLists.txt @@ -13,8 +13,13 @@ FetchContent_Declare( zstd GIT_REPOSITORY https://github.com/facebook/zstd.git GIT_TAG v1.5.6 + SOURCE_SUBDIR build/cmake ) +set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "" FORCE) +set(ZSTD_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(ZSTD_BUILD_SHARED OFF CACHE BOOL "" FORCE) +set(ZSTD_BUILD_STATIC ON CACHE BOOL "" FORCE) FetchContent_MakeAvailable(zstd) -target_link_libraries(zstd_lib photon_static zstd) +target_link_libraries(zstd_lib photon_static libzstd_static)