diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..12e2dcb --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1 @@ +FROM wasmedge/wasmedge:ci-image-base diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..e38f69d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,6 @@ +{ + "name": "WasmEdge devcontainer development", + "build": { + "dockerfile": "Dockerfile" + } +} diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4e4023c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +/build +/debug diff --git a/.github/workflows/build_for_openwrt.yml b/.github/workflows/build_for_openwrt.yml new file mode 100644 index 0000000..a4e0827 --- /dev/null +++ b/.github/workflows/build_for_openwrt.yml @@ -0,0 +1,135 @@ +name: Test WasmEdge on OpenWrt + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build_openwrt: + name: OpenWrt + runs-on: ubuntu-latest + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: Ensure git safe directory + run: | + git config --global --add safe.directory $(pwd) + - name: Download WasmEdge 0.18.1 source + run: | + curl -fsSL https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.18.1.tar.gz -o WasmEdge-0.18.1.tar.gz + mkdir -p WasmEdge + tar -xzf WasmEdge-0.18.1.tar.gz --strip-components=1 -C WasmEdge + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.9' + - name: Restore OpenWrt + id: restore-openwrt + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ./openwrt-x86-64-generic-rootfs.tar.gz + key: openwrt-${{ hashFiles('.github/workflows/build_for_openwrt.yml') }} + - name: Install dependency + if: steps.restore-openwrt.outputs.cache-hit != 'true' + run: | + git clone https://github.com/openwrt/openwrt.git ./openwrt + cd ./openwrt + git checkout 9dc86d19628f55bfed2048c412c5b23bda328cff + - name: Update feeds + if: steps.restore-openwrt.outputs.cache-hit != 'true' + run: | + cd ./openwrt + ./scripts/feeds update -a + ./scripts/feeds install -a + - name: Make WasmEdge package + if: steps.restore-openwrt.outputs.cache-hit != 'true' + run: | + cd . + mkdir -p ./openwrt/package/utils/WasmEdge + cp -r ./WasmEdge ./openwrt/package/utils/WasmEdge/src + cp ./WasmEdge/utils/openwrt/configuration/Makefile ./openwrt/package/utils/WasmEdge/ + cd ./openwrt + ls ./ + rm -f ./.config* + touch ./.config + cat >> .config <> ./Dockerfile <> $GITHUB_OUTPUT + echo "docker_image=$DOCKER_IMAGE" >> $GITHUB_OUTPUT + echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + + paths-filter: + name: Run paths-filter + runs-on: ubuntu-latest + + permissions: + pull-requests: read + + outputs: # Set outputs to true if not pull_request + ci-image-base: ${{ github.event_name == 'pull_request' && steps.filter.outputs.ci-image-base || 'true' }} + alpine-base: ${{ github.event_name == 'pull_request' && steps.filter.outputs.alpine-base || 'true' }} + manylinux: ${{ github.event_name == 'pull_request' && steps.filter.outputs.manylinux || 'true' }} + ubuntu: ${{ github.event_name == 'pull_request' && steps.filter.outputs.ubuntu || 'true' }} + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - if: ${{ github.event_name == 'pull_request' }} + id: filter + uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 + with: + filters: | + ci-image-base: + - '.github/workflows/docker.yml' + - 'docker-bake.ci-image-base.hcl' + - 'Dockerfile.ci-image-base' + alpine-base: + - '.github/workflows/docker.yml' + - 'docker-bake.alpine-base.hcl' + - 'Dockerfile.alpine-base' + manylinux: + - '.github/workflows/docker.yml' + - 'docker-bake.manylinux.hcl' + - 'Dockerfile.manylinux**' + - 'scripts/ffmpeg/**' + - 'scripts/opencvmini/**' + - 'scripts/wasi-crypto/**' + - 'scripts/wasi-nn/**' + ubuntu: + - '.github/workflows/docker.yml' + - 'docker-bake.ubuntu.hcl' + - 'Dockerfile.ubuntu-**' + - 'scripts/ffmpeg/**' + - 'scripts/opencvmini/**' + - 'scripts/wasi-crypto/**' + - 'scripts/wasi-nn/**' + + bake-base-images: + needs: + - prep + - paths-filter + + name: CI image base + runs-on: ubuntu-latest + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Login to DockerHub + if: github.event_name != 'pull_request' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + # DO NOT skip the whole job to prevent descending jobs to be skipped + - if: ${{ needs.paths-filter.outputs.ci-image-base == 'true' }} + name: Bake and Push + uses: docker/bake-action@59cb1203cd42bcefd40a1b6d73b589e48c86040e # v7.5.0 + with: + files: docker-bake.ci-image-base.hcl + push: ${{ github.event_name != 'pull_request' }} + + build-alpine-base-images: + needs: + - prep + - paths-filter + # Only run if files changed or if it's a scheduled/merge build + if: ${{ needs.paths-filter.outputs.alpine-base == 'true' }} + + name: Alpine Base (3.23) + runs-on: ubuntu-latest + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Login to DockerHub + if: github.event_name != 'pull_request' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + - name: Bake and Push + uses: docker/bake-action@59cb1203cd42bcefd40a1b6d73b589e48c86040e # v7.5.0 + with: + # Point to your new HCL file here + files: docker-bake.alpine-base.hcl + push: ${{ github.event_name != 'pull_request' }} + set: | + *.labels.org.opencontainers.image.title=${{ github.event.repository.name }} + *.labels.org.opencontainers.image.description=${{ github.event.repository.description }} + *.labels.org.opencontainers.image.url=${{ github.event.repository.html_url }} + *.labels.org.opencontainers.image.source=${{ github.event.repository.clone_url }} + *.labels.org.opencontainers.image.revision=${{ github.sha }} + *.labels.org.opencontainers.image.version=${{ needs.prep.outputs.version }} + *.labels.org.opencontainers.image.created=${{ needs.prep.outputs.created }} + + build-ubuntu-images: + needs: + - prep + - paths-filter + - bake-base-images + if: ${{ needs.paths-filter.outputs.ubuntu == 'true' }} + + strategy: + fail-fast: false + matrix: + targets: + - default + - base-2004-clang-aarch64 + + name: Ubuntu + runs-on: ubuntu-latest + container: + image: wasmedge/wasmedge:ci-image-base + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Login to DockerHub + if: github.event_name != 'pull_request' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + - name: Bake and Push + uses: docker/bake-action@59cb1203cd42bcefd40a1b6d73b589e48c86040e # v7.5.0 + with: + files: docker-bake.ubuntu.hcl + targets: ${{ matrix.targets }} + push: ${{ github.event_name != 'pull_request' }} + set: | + *.labels.org.opencontainers.image.title=${{ github.event.repository.name }} + *.labels.org.opencontainers.image.description=${{ github.event.repository.description }} + *.labels.org.opencontainers.image.url=${{ github.event.repository.html_url }} + *.labels.org.opencontainers.image.source=${{ github.event.repository.clone_url }} + *.labels.org.opencontainers.image.revision=${{ github.sha }} + *.labels.org.opencontainers.image.version=${{ needs.prep.outputs.version }} + *.labels.org.opencontainers.image.created=${{ needs.prep.outputs.created }} + + build-manylinux-images: + needs: + - prep + - paths-filter + - bake-base-images + if: ${{ needs.paths-filter.outputs.manylinux == 'true' }} + + strategy: + fail-fast: false + matrix: + include: + - name: manylinux_2_28 x86_64 + docker_tag: manylinux_2_28_x86_64 + targets: x86_64,x86_64-plugins + host_runner: ubuntu-latest + host_container: wasmedge/wasmedge:ci-image-base + - name: manylinux_2_28 aarch64 + docker_tag: manylinux_2_28_aarch64 + targets: aarch64,aarch64-plugins + host_runner: ubuntu-24.04-arm + host_container: wasmedge/wasmedge:ci-image-base_aarch64 + + name: ${{ matrix.name }} + runs-on: ${{ matrix.host_runner }} + container: + image: ${{ matrix.host_container }} + + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Login to DockerHub + if: github.event_name != 'pull_request' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_ACCESS_TOKEN }} + + - name: Bake and Push + uses: docker/bake-action@59cb1203cd42bcefd40a1b6d73b589e48c86040e # v7.5.0 + with: + files: docker-bake.manylinux.hcl + targets: ${{ matrix.targets }} + push: ${{ github.event_name != 'pull_request' }} + set: | + *.labels.org.opencontainers.image.title=${{ github.event.repository.name }} + *.labels.org.opencontainers.image.description=${{ github.event.repository.description }} + *.labels.org.opencontainers.image.url=${{ github.event.repository.html_url }} + *.labels.org.opencontainers.image.source=${{ github.event.repository.clone_url }} + *.labels.org.opencontainers.image.revision=${{ github.sha }} + *.labels.org.opencontainers.image.version=${{ needs.prep.outputs.version }} + *.labels.org.opencontainers.image.created=${{ needs.prep.outputs.created }} diff --git a/.github/workflows/reusable-build-on-alpine-static.yml b/.github/workflows/reusable-build-on-alpine-static.yml new file mode 100644 index 0000000..c7973d1 --- /dev/null +++ b/.github/workflows/reusable-build-on-alpine-static.yml @@ -0,0 +1,63 @@ +name: Build on Alpine (static lib) + +on: + workflow_call: + inputs: + version: + type: string + required: true + release: + type: boolean + +permissions: + contents: read + +jobs: + build_on_alpine_static: + permissions: + contents: write + name: Build on Alpine (static lib) + runs-on: ubuntu-latest + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: Ensure git safe directory + run: | + git config --global --add safe.directory $(pwd) + - name: Download WasmEdge 0.18.1 source + run: | + curl -fsSL https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.18.1.tar.gz -o WasmEdge-0.18.1.tar.gz + mkdir -p WasmEdge-0.18.1 + tar -xzf WasmEdge-0.18.1.tar.gz --strip-components=1 -C WasmEdge-0.18.1 + printf '0.18.1' > WasmEdge-0.18.1/VERSION + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Build WasmEdge + uses: docker/bake-action@59cb1203cd42bcefd40a1b6d73b589e48c86040e # v7.5.0 + with: + source: . + files: ./docker-bake.alpine-static.hcl + - name: Upload artifact + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: static-libs-alpine + path: ./build/*/WasmEdge-*.tar.gz + - name: Upload package tarball + if: ${{ inputs.release }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + type -p curl >/dev/null || (apt update && apt install curl -y) + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ + && chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt update \ + && apt install gh -y + gh release upload ${{ inputs.version }} ./build/*/WasmEdge-*.tar.gz --clobber diff --git a/.github/workflows/reusable-build-on-debian-static.yml b/.github/workflows/reusable-build-on-debian-static.yml new file mode 100644 index 0000000..617bcf8 --- /dev/null +++ b/.github/workflows/reusable-build-on-debian-static.yml @@ -0,0 +1,68 @@ +name: Build on Debian (static lib) + +on: + workflow_call: + inputs: + version: + type: string + required: true + release: + type: boolean + +permissions: + contents: read + +jobs: + build_on_debian_static: + permissions: + contents: write + name: Build on Debian (static lib) + runs-on: ubuntu-latest + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@ccd8616d44fd3846e67624a50d5aad6d37bf2d25 # v2.22.1 + with: + egress-policy: audit + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: Ensure git safe directory + run: | + git config --global --add safe.directory $(pwd) + - name: Download WasmEdge 0.18.1 source + run: | + curl -fsSL https://github.com/WasmEdge/WasmEdge/archive/refs/tags/0.18.1.tar.gz -o WasmEdge-0.18.1.tar.gz + mkdir -p WasmEdge-0.18.1 + tar -xzf WasmEdge-0.18.1.tar.gz --strip-components=1 -C WasmEdge-0.18.1 + printf '0.18.1' > WasmEdge-0.18.1/VERSION + - name: Set up QEMU + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + with: + image: docker.io/tonistiigi/binfmt@sha256:400a4873b838d1b89194d982c45e5fb3cda4593fbfd7e08a02e76b03b21166f0 + platforms: arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + - name: Build WasmEdge + uses: docker/bake-action@59cb1203cd42bcefd40a1b6d73b589e48c86040e # v7.5.0 + with: + source: . + files: ./docker-bake.debian-static.hcl + - name: Upload artifact + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: static-libs-debian + path: ./build/*/WasmEdge-*.tar.gz + - name: Upload package tarball + if: ${{ inputs.release }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + type -p curl >/dev/null || (apt update && apt install curl -y) + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg \ + && chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \ + && apt update \ + && apt install gh -y + gh release upload ${{ inputs.version }} ./build/*/WasmEdge-*.tar.gz --clobber diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..74d5412 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +docs/superpowers/ diff --git a/Dockerfile.alpine-base b/Dockerfile.alpine-base new file mode 100644 index 0000000..c1e587e --- /dev/null +++ b/Dockerfile.alpine-base @@ -0,0 +1,62 @@ +# syntax=docker/dockerfile:1.5-labs + +ARG ALPINE_VERSION=3.23 + +# ---------------------------------------------------- +# STAGE 1: Builder +# Compiles LLD from source, linking against system LLVM +# ---------------------------------------------------- +FROM alpine:${ALPINE_VERSION} AS builder + +# Use a specific tag for reproducibility +# Pinned version to match Alpine 3.23's system LLVM (v19) +ARG LLVM_VERSION=llvmorg-19.1.5 + +# Build Dependencies: +# - build-base: Compiler toolchain (gcc, g++, make, libc-dev) required for compilation. +# - cmake: The build system used by LLVM. +# - ninja: The build generator (faster than make). +# - python3: Required by LLVM build scripts and configuration utilities. +# - git: Required to clone the LLVM repository. +# - linux-headers: Ensures system header compatibility for C++ standard library builds. +# - llvm19-dev: Provides LLVM development headers for version 19. +# - llvm19-static: Provides static LLVM libraries for version 19. +# - zstd-static: Provides static zstd compression libraries. +# - zlib-static: Provides static zlib compression libraries. +RUN apk add --no-cache build-base cmake ninja python3 git linux-headers \ + llvm19-dev llvm19-static zstd-static zlib-static + + +# The system CMake files require testing libraries that are missing in the Alpine package. +# We create empty dummy archives to satisfy the dependency check. +RUN LLVM_LIB_DIR=/usr/lib/llvm19/lib && \ + for f in libLLVMTestingAnnotations.a libLLVMTestingSupport.a libllvm_gtest.a libllvm_gtest_main.a; do \ + ar rc "$LLVM_LIB_DIR/$f"; \ + done + +# Clone LLVM Project +WORKDIR /src +# Shallow clone specific tag +RUN git clone --depth 1 -b ${LLVM_VERSION} https://github.com/llvm/llvm-project.git + + +# Configure and Build LLD +# We treat LLD as a standalone project and point it to the system LLVM +WORKDIR /src/llvm-project/lld +RUN cmake -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DLLVM_DIR=/usr/lib/llvm19/lib/cmake/llvm \ + -DLLVM_BUILD_STATIC=ON \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF && \ + cmake --build build --target lldELF lldCommon lldMachO lldCOFF lldWasm lldMinGW + +# ---------------------------------------------------- +# STAGE 2: Final Image (Contains only the compiled static libraries) +# ---------------------------------------------------- +FROM alpine:${ALPINE_VERSION} + +# Copy artifacts to a standard library path +COPY --from=builder /src/llvm-project/lld/build/lib/liblld*.a /usr/local/lib/ + diff --git a/Dockerfile.alpine-static b/Dockerfile.alpine-static new file mode 100644 index 0000000..1801446 --- /dev/null +++ b/Dockerfile.alpine-static @@ -0,0 +1,116 @@ +# syntax=docker/dockerfile:1.5-labs + +ARG XX_VERSION=1.2.1 +ARG ALPINE_VERSION=3.23 + + +# ---------------------------------------------------- +# STAGE 1: Import Pre-Built LLD Libraries +# ---------------------------------------------------- +FROM wasmedge/wasmedge:alpine-base-3.23 AS lld-artifact + +# ---------------------------------------------------- +# STAGE 2: Main WasmEdge Build +# ---------------------------------------------------- +FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx +FROM --platform=$BUILDPLATFORM alpine:${ALPINE_VERSION} AS base +COPY --from=xx / / + +# Install host dependencies +RUN apk add bash cmake samurai g++ clang +SHELL [ "bash", "-c" ] + +# Make a cmake toolchain file +RUN cat <<'EOT' > /usr/bin/xx-toolchain && chmod a+x /usr/bin/xx-toolchain +#!/bin/bash +mkdir -p /etc/xx-toolchains/ +TOOLCHAIN_FILE="/etc/xx-toolchains/$(xx-clang --print-target-triple).cmake" +[ -f "$TOOLCHAIN_FILE" ] || cat < "$TOOLCHAIN_FILE" +set(CMAKE_CROSSCOMPILING ON) +set(CMAKE_SYSROOT "$(xx-info sysroot)") +set(CMAKE_SYSTEM_NAME "Linux") +set(CMAKE_SYSTEM_VERSION 1) +set(CMAKE_SYSTEM_PROCESSOR "$(xx-info march)") +set(CMAKE_C_COMPILER "$(which clang)") +set(CMAKE_CXX_COMPILER "$(which clang++)") +set(CMAKE_ASM_COMPILER "$(which clang)") +set(CMAKE_AR "$(which ar)") +set(CMAKE_RANLIB "$(which ranlib)") +set(PKG_CONFIG_EXECUTABLE "$(xx-clang --print-prog-name=pkg-config)") +set(CMAKE_C_COMPILER_TARGET "$(xx-clang --print-target-triple)") +set(CMAKE_CXX_COMPILER_TARGET "$(xx-clang --print-target-triple)") +set(CMAKE_ASM_COMPILER_TARGET "$(xx-clang --print-target-triple)") +EOF +echo "$TOOLCHAIN_FILE" +EOT + +FROM base AS config +ARG TARGETPLATFORM + +RUN apk add git llvm-dev +# We add zstd-static because the SYSTEM LLVM (installed via apk) depends on it. +RUN xx-apk add \ + g++ \ + llvm-dev llvm-static \ + lld lld-dev \ + zlib-dev zlib-static \ + zstd-static + +# Fix LLVMConfig.cmake to account for the sysroot +RUN sed -i 's|/usr/lib/llvm|${CMAKE_SYSROOT}usr/lib/llvm|' $(xx-info sysroot)usr/lib/cmake/llvm*/LLVMConfig.cmake + +# Patch: Create dummy files for missing testing libraries to satisfy CMake +RUN for d in $(xx-info sysroot)usr/lib/llvm*/lib; do \ + for f in libLLVMTestingAnnotations.a libLLVMTestingSupport.a libllvm_gtest.a libllvm_gtest_main.a; do \ + ar rc "$d/$f"; \ + done \ + done + +# Inject the Native LLD libraries from Stage 1 (The Base Image) +# Note: We changed the source path to /usr/local/lib because that's where we put them in the base image +COPY --from=lld-artifact /usr/local/lib/liblld*.a /tmp/lld-libs/ +RUN cp /tmp/lld-libs/*.a $(xx-info sysroot)usr/lib/llvm*/lib/ + +# Use the native version of llvm-config +RUN ! xx-info is-cross || cp -f /usr/lib/llvm*/bin/llvm-config $(xx-info sysroot)usr/lib/llvm*/bin/llvm-config + +RUN --mount=type=bind,target=/src,from=source \ + cmake -S /src -B /build -G Ninja \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX="/install" \ + # For cross compiling + -DCMAKE_TOOLCHAIN_FILE="$(xx-toolchain)" \ + -DWASMEDGE_BUILD_PACKAGE="TGZ" \ + -DWASMEDGE_USE_LLVM=ON \ + # Build just what we need + -DWASMEDGE_BUILD_STATIC_LIB=ON \ + -DWASMEDGE_BUILD_TESTS=OFF \ + -DWASMEDGE_BUILD_SHARED_LIB=OFF \ + -DWASMEDGE_BUILD_TOOLS=OFF \ + -DWASMEDGE_BUILD_PLUGINS=OFF \ + -DWASMEDGE_BUILD_EXAMPLE=OFF \ + # link llvm statically + -DWASMEDGE_LINK_LLVM_STATIC=ON \ + -DWASMEDGE_LINK_TOOLS_STATIC=ON \ + # Disable extra dependencies + -DWASMEDGE_DISABLE_LIBTINFO=ON + +FROM config AS build +RUN --mount=type=bind,target=/src,from=source \ + cmake --build /build -- install package + +ARG ALPINE_VERSION +RUN --mount=type=bind,target=/src,from=source < /etc/apt/trusted.gpg.d/apt.llvm.org.asc + echo "deb http://apt.llvm.org/$(lsb_release -sc)/ llvm-toolchain-$(lsb_release -sc)-${LLVM_VERSION} main" > /etc/apt/sources.list.d/llvm.list + apt-get update -y +EOT + +FROM base as deps +ARG TARGETPLATFORM LLVM_VERSION + +# Install llvm-*-dev: +RUN /bin/bash < package depends on llvm-*: and llvm-*-tools: but + # should depend on llvm-*-tools: and llvm-*:. + # Patch llvm-*-dev: to replace those dependencies. + # See: https://groups.google.com/g/linux.debian.bugs.dist/c/OrHgd5vY278 + cd $(mktemp -d) + xx-apt show llvm-${LLVM_VERSION}-dev + apt-get download llvm-${LLVM_VERSION}-dev:$(xx-info debian-arch) + ar x llvm-${LLVM_VERSION}-dev_*.deb + tar xJf control.tar.* + sed -Ei 's|(llvm-[0-9]*(-tools)?) |\1:'$(TARGETPLATFORM='' TARGETPAIR='' TARGETOS='' TARGETARCH='' TARGETVARIANT='' xx-info debian-arch)' |g' control + tar --ignore-failed-read -czf control.tar.gz {post,pre}{inst,rm} md5sums control + ar rcs llvm-dev-patched.deb debian-binary control.tar.gz data.tar.* + apt-get install --no-install-recommends -y ./llvm-dev-patched.deb +EOT + +# Install other *: dev dependencies +RUN xx-apt-get install --no-install-recommends -y \ + xx-cxx-essentials \ + liblld-${LLVM_VERSION}-dev libpolly-${LLVM_VERSION}-dev \ + libncurses-dev zlib1g-dev libzstd-dev + +# Make a cmake toolchain file +RUN cat < /toolchain.cmake + set(CMAKE_SYSTEM_NAME "Linux") + set(CMAKE_SYSTEM_VERSION 1) + set(CMAKE_SYSTEM_PROCESSOR "$(xx-info march)") + set(CMAKE_C_COMPILER "xx-clang") + set(CMAKE_CXX_COMPILER "xx-clang++") + set(CMAKE_ASM_COMPILER "xx-clang") + set(CMAKE_AR "ar") + set(DPKG_CONFIG_EXECUTABLE $(xx-clang --print-prog-name=pkg-config)) + set(DCMAKE_C_COMPILER_TARGET $(xx-clang --print-target-triple)) + set(DCMAKE_CXX_COMPILER_TARGET $(xx-clang --print-target-triple)) + set(DCMAKE_ASM_COMPILER_TARGET $(xx-clang --print-target-triple)) +EOT + +FROM deps as src +COPY --from=source . /src + +FROM src as build +RUN cmake -S /src -B /build -G Ninja \ + -DCMAKE_TOOLCHAIN_FILE=/toolchain.cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_INSTALL_PREFIX=/install \ + -DWASMEDGE_BUILD_PACKAGE="TGZ" \ + -DWASMEDGE_BUILD_TESTS=OFF \ + -DWASMEDGE_BUILD_SHARED_LIB=OFF \ + -DWASMEDGE_BUILD_STATIC_LIB=ON \ + -DWASMEDGE_BUILD_TOOLS=OFF \ + -DWASMEDGE_BUILD_PLUGINS=OFF \ + -DWASMEDGE_BUILD_EXAMPLE=OFF \ + -DWASMEDGE_LINK_LLVM_STATIC=ON \ + -DWASMEDGE_LINK_TOOLS_STATIC=ON \ + -DWASMEDGE_USE_LLVM=ON + +RUN cmake --build /build -- install +RUN cmake --build /build -- package +RUN mv /build/WasmEdge-*-Linux.tar.gz $(echo build/WasmEdge-*-Linux.tar.gz | sed 's|WasmEdge-\(.*\)-Linux.tar.gz|WasmEdge-\1-debian'$(lsb_release -sr)'_'$(xx-info march)'_static.tar.gz|') + +FROM scratch as tar +COPY --from=build /build/WasmEdge-*.tar.gz / diff --git a/Dockerfile.manylinux2014-build-plugins-deps b/Dockerfile.manylinux2014-build-plugins-deps new file mode 100644 index 0000000..d233859 --- /dev/null +++ b/Dockerfile.manylinux2014-build-plugins-deps @@ -0,0 +1,38 @@ +ARG BASE=wasmedge/wasmedge:manylinux2014_x86_64 +FROM ${BASE} + +ENV PATH /opt/rh/devtoolset-11/root/usr/bin${PATH:+:${PATH}} +ENV MANPATH /opt/rh/devtoolset-11/root/usr/share/man${MANPATH:+:${MANPATH}} +ENV INFOPATH /opt/rh/devtoolset-11/root/usr/share/info${INFOPATH:+:${INFOPATH}} +ENV PKG_CONFIG_PATH /opt/rh/devtoolset-11/root/usr/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} + +RUN cd && (yum check-update || true) && \ + yum install -y cmake wget unzip zlib-devel zlib-static +RUN yum-config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo && \ + yum install -y gh + +WORKDIR /root + +COPY scripts/opencvmini/install-opencvmini.sh . +ENV OPENCV_VERSION "4.8.0" +RUN [ "/bin/bash", "install-opencvmini.sh" ] + +COPY scripts/wasi-nn/install-pytorch.sh . +ENV PYTORCH_VERSION "2.5.1" +ENV PYTORCH_INSTALL_TO "/root" +ENV Torch_DIR "/root/libtorch" +RUN [ "/bin/bash", "install-pytorch.sh", "--disable-cxx11-abi" ] + +COPY scripts/wasi-crypto/build-openssl.sh . +ENV OPENSSL_ROOT_DIR "/root/openssl-1.1.1n/openssl" +RUN [ "/bin/bash", "build-openssl.sh" ] + +COPY scripts/ffmpeg/install-ffmpeg-v7.1.1.sh . +RUN [ "/bin/bash", "install-ffmpeg-v7.1.1.sh" ] +ENV PKG_CONFIG_PATH /root/FFmpeg-n7.1.1/output/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} +ENV LD_LIBRARY_PATH /root/FFmpeg-n7.1.1/output/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} + +ENV OPENVINO_VERSION "2025.0.0" +ENV OPENVINO_YEAR "2025" + +RUN yum clean all diff --git a/Dockerfile.manylinux2014_aarch64 b/Dockerfile.manylinux2014_aarch64 new file mode 100644 index 0000000..b5c45af --- /dev/null +++ b/Dockerfile.manylinux2014_aarch64 @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors + +FROM quay.io/pypa/manylinux2014_aarch64 + +MAINTAINER hydai hydai@secondstate.io + +ADD SHA256SUM.manylinux2014 /root/ + +ENV PATH /opt/rh/devtoolset-10/root/usr/bin${PATH:+:${PATH}} +ENV MANPATH /opt/rh/devtoolset-10/root/usr/share/man${MANPATH:+:${MANPATH}} +ENV INFOPATH /opt/rh/devtoolset-10/root/usr/share/info${INFOPATH:+:${INFOPATH}} +ENV PKG_CONFIG_PATH /opt/rh/devtoolset-10/root/usr/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} + +RUN cd && (yum check-update || true) && yum install -y xz openssl-devel rpm-build centos-release-scl && \ + yum install -y devtoolset-10 && \ + export CPU=$(/opt/python/cp311-cp311/bin/python3 -c \ + 'import multiprocessing; print(multiprocessing.cpu_count())') && \ + export CFGFLAGS="--prefix=/opt/rh/devtoolset-10/root/usr --disable-shared --libdir=/opt/rh/devtoolset-10/root/usr/lib64" && \ + curl -s -L -O --remote-name-all \ + https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz \ + https://github.com/Kitware/CMake/releases/download/v3.31.2/cmake-v3.31.2.tar.gz \ + https://github.com/ninja-build/ninja/archive/refs/tags/v1.12.1.tar.gz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/llvm-19.1.5.src.tar.xz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/lld-19.1.5.src.tar.xz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/libunwind-19.1.5.src.tar.xz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/cmake-19.1.5.src.tar.xz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/third-party-19.1.5.src.tar.xz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/clang-19.1.5.src.tar.xz && \ + sha256sum -c SHA256SUM.manylinux2014 && \ + gzip -dc zstd-1.5.6.tar.gz | tar -xf - && \ + gzip -dc cmake-v3.31.2.tar.gz | tar -xf - && \ + gzip -dc v1.12.1.tar.gz | tar -xf - && \ + xz -dc llvm-19.1.5.src.tar.xz | tar -xf - && \ + xz -dc lld-19.1.5.src.tar.xz | tar -xf - && \ + xz -dc libunwind-19.1.5.src.tar.xz | tar -xf - && \ + xz -dc cmake-19.1.5.src.tar.xz | tar -xf - && \ + xz -dc third-party-19.1.5.src.tar.xz | tar -xf - && \ + xz -dc clang-19.1.5.src.tar.xz | tar -xf - && \ + export ZSTDFLAGS=(PREFIX=/opt/rh/devtoolset-10/root/usr LIBDIR=/opt/rh/devtoolset-10/root/usr/lib64 SED_ERE_OPT=--regexp-extended MOREFLAGS="-std=c17 -O3 -fPIC -fPIE -fvisibility=hidden") && \ + cd zstd-1.5.6 && make -s "${ZSTDFLAGS[@]}" -j $CPU && make -s "${ZSTDFLAGS[@]}" install && rm -vf /opt/rh/devtoolset-10/root/usr/lib64/libzstd.so* && cd - && \ + mkdir build && cd build && /opt/python/cp311-cp311/bin/python \ + ../ninja-1.12.1/configure.py --bootstrap \ + --with-python=/opt/python/cp311-cp311/bin/python && \ + cp -v ninja /opt/rh/devtoolset-10/root/usr/bin/ninja && cd - && rm -rf build && \ + mkdir build && cd build && ../cmake-v3.31.2/configure --prefix=/opt/rh/devtoolset-10/root/usr \ + --parallel=$CPU && make -s -j $CPU && make -s install && cd - && rm -rf build && \ + mv -v llvm-19.1.5.src llvm && \ + mv -v lld-19.1.5.src lld && \ + mv -v libunwind-19.1.5.src libunwind && \ + mv -v cmake-19.1.5.src cmake && \ + mv -v third-party-19.1.5.src third-party && \ + mv -v clang-19.1.5.src clang && \ + cmake -Bbuild -GNinja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/rh/devtoolset-10/root/usr \ + -DPython3_ROOT_DIR=/opt/python/cp311-cp311 -DLLVM_LIBDIR_SUFFIX=64 \ + -DLLVM_TARGETS_TO_BUILD="AArch64;BPF" -DLLVM_ENABLE_PROJECTS="lld;clang" \ + -DLLVM_DEFAULT_TARGET_TRIPLE="aarch64-redhat-linux-gnu" \ + -DBUILD_SHARED_LIBS=OFF llvm && \ + cmake --build build --target install && \ + rm -rf build && rm -rf * + +RUN yum clean all diff --git a/Dockerfile.manylinux2014_x86_64 b/Dockerfile.manylinux2014_x86_64 new file mode 100644 index 0000000..e4277cb --- /dev/null +++ b/Dockerfile.manylinux2014_x86_64 @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors + +FROM quay.io/pypa/manylinux2014_x86_64 + +MAINTAINER hydai hydai@secondstate.io + +ADD SHA256SUM.manylinux2014 /root/ + +ENV PATH /opt/rh/devtoolset-11/root/usr/bin${PATH:+:${PATH}} +ENV MANPATH /opt/rh/devtoolset-11/root/usr/share/man${MANPATH:+:${MANPATH}} +ENV INFOPATH /opt/rh/devtoolset-11/root/usr/share/info${INFOPATH:+:${INFOPATH}} +ENV PKG_CONFIG_PATH /opt/rh/devtoolset-11/root/usr/lib64/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} + +RUN cd && (yum check-update || true) && yum install -y xz openssl-devel rpm-build dpkg centos-release-scl && \ + yum install -y devtoolset-11 && \ + export CPU=$(/opt/python/cp311-cp311/bin/python3 -c \ + 'import multiprocessing; print(multiprocessing.cpu_count())') && \ + export CFGFLAGS="--prefix=/opt/rh/devtoolset-11/root/usr --disable-shared --libdir=/opt/rh/devtoolset-11/root/usr/lib64" && \ + curl -s -L -O --remote-name-all \ + https://github.com/facebook/zstd/releases/download/v1.5.6/zstd-1.5.6.tar.gz \ + https://github.com/Kitware/CMake/releases/download/v3.31.2/cmake-v3.31.2.tar.gz \ + https://github.com/ninja-build/ninja/archive/refs/tags/v1.12.1.tar.gz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/llvm-19.1.5.src.tar.xz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/lld-19.1.5.src.tar.xz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/libunwind-19.1.5.src.tar.xz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/cmake-19.1.5.src.tar.xz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/third-party-19.1.5.src.tar.xz \ + https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.5/clang-19.1.5.src.tar.xz && \ + sha256sum -c SHA256SUM.manylinux2014 && \ + gzip -dc zstd-1.5.6.tar.gz | tar -xf - && \ + gzip -dc cmake-v3.31.2.tar.gz | tar -xf - && \ + gzip -dc v1.12.1.tar.gz | tar -xf - && \ + xz -dc llvm-19.1.5.src.tar.xz | tar -xf - && \ + xz -dc lld-19.1.5.src.tar.xz | tar -xf - && \ + xz -dc libunwind-19.1.5.src.tar.xz | tar -xf - && \ + xz -dc cmake-19.1.5.src.tar.xz | tar -xf - && \ + xz -dc third-party-19.1.5.src.tar.xz | tar -xf - && \ + xz -dc clang-19.1.5.src.tar.xz | tar -xf - && \ + export ZSTDFLAGS=(PREFIX=/opt/rh/devtoolset-11/root/usr LIBDIR=/opt/rh/devtoolset-11/root/usr/lib64 SED_ERE_OPT=--regexp-extended MOREFLAGS="-std=c17 -O3 -fPIC -fPIE -fvisibility=hidden") && \ + cd zstd-1.5.6 && make -s "${ZSTDFLAGS[@]}" -j $CPU && make -s "${ZSTDFLAGS[@]}" install && rm -vf /opt/rh/devtoolset-11/root/usr/lib64/libzstd.so* && cd - && \ + mkdir build && cd build && /opt/python/cp311-cp311/bin/python \ + ../ninja-1.12.1/configure.py --bootstrap \ + --with-python=/opt/python/cp311-cp311/bin/python && \ + cp -v ninja /opt/rh/devtoolset-11/root/usr/bin/ninja && cd - && rm -rf build && \ + mkdir build && cd build && ../cmake-v3.31.2/configure --prefix=/opt/rh/devtoolset-11/root/usr \ + --parallel=$CPU && make -s -j $CPU && make -s install && cd - && rm -rf build && \ + mv -v llvm-19.1.5.src llvm && \ + mv -v lld-19.1.5.src lld && \ + mv -v libunwind-19.1.5.src libunwind && \ + mv -v cmake-19.1.5.src cmake && \ + mv -v third-party-19.1.5.src third-party && \ + mv -v clang-19.1.5.src clang && \ + cmake -Bbuild -GNinja -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/rh/devtoolset-11/root/usr \ + -DPython3_ROOT_DIR=/opt/python/cp311-cp311 -DLLVM_LIBDIR_SUFFIX=64 \ + -DLLVM_TARGETS_TO_BUILD="X86;BPF" -DLLVM_ENABLE_PROJECTS="lld;clang" \ + -DLLVM_DEFAULT_TARGET_TRIPLE="x86_64-pc-linux-gnu" \ + -DBUILD_SHARED_LIBS=OFF llvm && \ + cmake --build build --target install && \ + rm -rf build && rm -rf * + +RUN yum clean all diff --git a/Dockerfile.manylinux_2_28-base b/Dockerfile.manylinux_2_28-base index c7fee27..64d7025 100644 --- a/Dockerfile.manylinux_2_28-base +++ b/Dockerfile.manylinux_2_28-base @@ -1,9 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# SPDX-FileCopyrightText: 2019-2024 Second State INC +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors ARG BASE_IMAGE="quay.io/pypa/manylinux_2_28_x86_64" FROM ${BASE_IMAGE} -ADD scripts/SHA256SUM.manylinux_2_28 /root/ +ADD SHA256SUM.manylinux_2_28 /root/ # See /opt/rh/gcc-toolset-13/enable ENV PATH=/opt/rh/gcc-toolset-13/root/usr/bin${PATH:+:${PATH}} diff --git a/Dockerfile.manylinux_2_28-plugins-deps b/Dockerfile.manylinux_2_28-plugins-deps index 3362f32..4ecc921 100644 --- a/Dockerfile.manylinux_2_28-plugins-deps +++ b/Dockerfile.manylinux_2_28-plugins-deps @@ -6,18 +6,23 @@ WORKDIR /root ### deps for x86_64 ### FROM base AS deps-amd64 RUN cd && (yum check-update || true) && \ - yum install -y wget unzip zlib-devel zlib-static elfutils-libelf-devel + yum install -y wget unzip zlib-devel zlib-static elfutils-libelf-devel zstd COPY scripts/wasi-nn/install-pytorch.sh . -ENV PYTORCH_VERSION="2.5.1" -ENV PYTORCH_INSTALL_TO="/root" +# The pre-cxx11-abi libtorch is kept for the branches still building with the +# pre-cxx11 ABI; remove it after the 0.18.0 release. +ENV PYTORCH_INSTALL_TO="/root/libtorch" ENV Torch_DIR="/root/libtorch" RUN [ "/bin/bash", "install-pytorch.sh", "--disable-cxx11-abi" ] +ENV PYTORCH_VERSION="2.12.1" +ENV PYTORCH_INSTALL_TO="/root/libtorch-cxx11abi" +RUN [ "/bin/bash", "install-pytorch.sh" ] + ### deps for aarch64 ### FROM base AS deps-arm64 RUN cd && (yum check-update || true) && \ - yum install -y wget unzip zlib-devel zlib-static + yum install -y wget unzip zlib-devel zlib-static zstd ### deps for all ### FROM deps-${TARGETARCH} AS final @@ -30,10 +35,10 @@ COPY scripts/wasi-crypto/build-openssl.sh . ENV OpenSSL_DIR="/root/openssl-1.1.1n/openssl" RUN [ "/bin/bash", "build-openssl.sh" ] -COPY scripts/ffmpeg/install-ffmpeg-v6.0.sh . -RUN [ "/bin/bash", "install-ffmpeg-v6.0.sh" ] -ENV PKG_CONFIG_PATH=/root/FFmpeg-n6.0/output/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} -ENV LD_LIBRARY_PATH=/root/FFmpeg-n6.0/output/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} +COPY scripts/ffmpeg/install-ffmpeg-v7.1.1.sh . +RUN [ "/bin/bash", "install-ffmpeg-v7.1.1.sh" ] +ENV PKG_CONFIG_PATH=/root/FFmpeg-n7.1.1/output/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} +ENV LD_LIBRARY_PATH=/root/FFmpeg-n7.1.1/output/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} ENV OPENVINO_VERSION="2025.0.0" ENV OPENVINO_YEAR="2025" @@ -41,4 +46,8 @@ ENV OPENVINO_YEAR="2025" COPY scripts/wasi-nn/install-onnxruntime.sh . RUN [ "/bin/bash", "install-onnxruntime.sh" ] +COPY scripts/wasi-nn/libpiper.patch . +COPY scripts/wasi-nn/install-libpiper.sh . +RUN [ "/bin/bash", "install-libpiper.sh" ] + RUN yum clean all diff --git a/Dockerfile.release b/Dockerfile.release new file mode 100644 index 0000000..ab05caf --- /dev/null +++ b/Dockerfile.release @@ -0,0 +1,13 @@ +FROM ubuntu:22.04 +ARG VERSION + +RUN apt-get update && apt-get install -y netbase +ADD WasmEdge-$VERSION-Linux.tar.gz /tmp/ +RUN cp -rf /tmp/WasmEdge-$VERSION-Linux/bin/* /usr/local/bin && \ + cp -rf /tmp/WasmEdge-$VERSION-Linux/lib64/* /usr/local/lib && \ + cp -rf /tmp/WasmEdge-$VERSION-Linux/include/* /usr/local/include && \ + ldconfig /usr/local/lib +RUN rm -rf /tmp/WasmEdge-$VERSION-Linux + +WORKDIR /app +CMD ["/usr/local/bin/wasmedge"] diff --git a/Dockerfile.ubuntu-base b/Dockerfile.ubuntu-base index 8ef2aa4..0fc6118 100644 --- a/Dockerfile.ubuntu-base +++ b/Dockerfile.ubuntu-base @@ -13,7 +13,8 @@ RUN apt-get update && \ ninja-build \ software-properties-common \ wget \ - zlib1g-dev + zlib1g-dev \ + zstd ### deps for ubuntu 20.04 ### FROM base AS deps-20 @@ -40,6 +41,7 @@ ENV CXX=/usr/bin/clang++-18 ### deps for ubuntu 22.04 ### FROM base AS deps-22 +RUN curl -sSf https://apt.kitware.com/kitware-archive.sh | sh RUN apt-get install -y cmake RUN apt-get install -y \ diff --git a/Dockerfile.ubuntu-plugins-deps b/Dockerfile.ubuntu-plugins-deps index cb745fb..a4b843a 100644 --- a/Dockerfile.ubuntu-plugins-deps +++ b/Dockerfile.ubuntu-plugins-deps @@ -30,15 +30,15 @@ RUN apt-get install -y \ libswresample-dev \ libswscale-dev -# FFmpeg 6.0 (ubuntu 20.04, 22.04) +# FFmpeg 7.1.1 (ubuntu 20.04, 22.04) FROM base AS deps-20 WORKDIR /usr/local -COPY scripts/ffmpeg/install-ffmpeg-v6.0.sh . -RUN [ "/bin/bash", "install-ffmpeg-v6.0.sh" ] -ENV PKG_CONFIG_PATH=/usr/local/FFmpeg-n6.0/output/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} -ENV LD_LIBRARY_PATH=/usr/local/FFmpeg-n6.0/output/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} +COPY scripts/ffmpeg/install-ffmpeg-v7.1.1.sh . +RUN [ "/bin/bash", "install-ffmpeg-v7.1.1.sh" ] +ENV PKG_CONFIG_PATH=/usr/local/FFmpeg-n7.1.1/output/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}} +ENV LD_LIBRARY_PATH=/usr/local/FFmpeg-n7.1.1/output/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} FROM deps-20 AS deps-22 @@ -52,8 +52,8 @@ ENV OPENCV_VERSION="4.8.0" RUN [ "/bin/bash", "install-opencvmini.sh" ] COPY scripts/wasi-nn/install-pytorch.sh . -ENV PYTORCH_VERSION="2.5.1" -ENV PYTORCH_INSTALL_TO="/usr/local" +ENV PYTORCH_VERSION="2.12.1" +ENV PYTORCH_INSTALL_TO="/usr/local/libtorch" ENV Torch_DIR="/usr/local/libtorch" RUN [ "/bin/bash", "install-pytorch.sh" ] @@ -75,6 +75,10 @@ RUN [ "/bin/bash", "-c", "echo \"source ./openvino_genai/setupvars.sh\" >> .bash COPY scripts/wasi-nn/install-onnxruntime.sh . RUN [ "/bin/bash", "install-onnxruntime.sh" ] +COPY scripts/wasi-nn/libpiper.patch . +COPY scripts/wasi-nn/install-libpiper.sh . +RUN [ "/bin/bash", "install-libpiper.sh" ] + COPY scripts/wasi-nn/install-chattts.sh . RUN [ "/bin/bash", "install-chattts.sh" ] @@ -83,11 +87,12 @@ FROM deps-all AS clean-apt RUN rm -f \ install-opencvmini.sh \ - install-ffmpeg-v6.0.sh \ + install-ffmpeg-v7.1.1.sh \ install-pytorch.sh \ install-openvino.sh \ install-onnxruntime.sh \ install-openvino-genai.sh \ - install-chattts.sh + install-chattts.sh \ + install-libpiper.sh RUN rm -rf /var/lib/apt/lists/* diff --git a/Dockerfile.ubuntu2004_x86_64 b/Dockerfile.ubuntu2004_x86_64 new file mode 100644 index 0000000..f71ed3c --- /dev/null +++ b/Dockerfile.ubuntu2004_x86_64 @@ -0,0 +1,24 @@ +FROM ubuntu:20.04 + +MAINTAINER hydai hydai@secondstate.io +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update && apt upgrade -y \ + && apt install -y \ + software-properties-common \ + wget \ + cmake \ + ninja-build \ + curl \ + git \ + dpkg-dev \ + llvm-12-dev \ + liblld-12-dev \ + gcc \ + rpm \ + g++ + +RUN rm -rf /var/lib/apt/lists/* + +ENV CC=gcc +ENV CXX=g++ diff --git a/Dockerfile.ubuntu2104_armv7l b/Dockerfile.ubuntu2104_armv7l new file mode 100644 index 0000000..e3e7b54 --- /dev/null +++ b/Dockerfile.ubuntu2104_armv7l @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors + +FROM arm32v7/ubuntu:hirsute + +MAINTAINER hydai hydai@secondstate.io +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt update && apt upgrade -y \ + && apt install -y \ + build-essential \ + cmake \ + curl \ + dpkg-dev \ + gcc \ + gcc-multilib \ + g++ \ + g++-multilib \ + git \ + llvm-12-dev \ + liblld-12-dev \ + libssl-dev \ + ninja-build \ + software-properties-common \ + python3 \ + rpm \ + wget \ + xz-utils + +# Build CMake from source to avoid compiler_id_detection failures when using QEMU user-mode emulation +# See: https://gitlab.kitware.com/cmake/cmake/-/issues/20568 + +#RUN wget https://github.com/Kitware/CMake/releases/download/v3.29.3/cmake-3.29.3.tar.gz --no-check-certificate && \ +# tar zxvf cmake-3.29.3.tar.gz && \ +# cd cmake-3.29.3 && \ +# ./configure && \ +# make install -j$(nproc) && \ +# cd .. && rm -rf cmake-3.29.3 + +RUN rm -rf /var/lib/apt/lists/* + +ENV CC=gcc +ENV CXX=g++ diff --git a/README.md b/README.md index 67c31a4..11d2918 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,20 @@ -# devecontainer for WasmEdge +# Devcontainer for WasmEdge -This repository contains dockerfiles and workflows for all devcontainers used by WasmEdge. +> [!WARNING] +> **Work in Progress** -**NOTE**: Work In Progress +This repository owns the Dockerfiles, dependency scripts, Bake definitions, +and GitHub Actions workflows used to build and validate WasmEdge development, +CI, static-library, and OpenWrt images. + +## Layout + +| Path | Contents | +|---|---| +| `.devcontainer/` | Development environment for `WasmEdge/devcontainer`. | +| `Dockerfile.*` | WasmEdge development, CI, plugin, static, legacy, and release image definitions. | +| `docker-bake.*.hcl` | Docker Buildx Bake definitions for the actively built image families. | +| `SHA256SUM.*` | Checksums used by the manylinux image definitions. | +| `build-manylinux.sh` | Legacy manylinux image build helper. | +| `scripts/` | Dockerfile-consumed dependency installers, build helpers, test data helpers, and patches, grouped by dependency family. | +| `.github/workflows/` | Image, static-library, and OpenWrt build and validation workflows. | diff --git a/SHA256SUM.manylinux2014 b/SHA256SUM.manylinux2014 new file mode 100644 index 0000000..63aa74f --- /dev/null +++ b/SHA256SUM.manylinux2014 @@ -0,0 +1,9 @@ +e7dfc8050407b5cc564c1c1afe19517255c9229cccd886dbd5bac9b652828d85 clang-19.1.5.src.tar.xz +a08ae477571fd5e929c27d3d0d28c6168d58dd00b6354c2de3266ae0d86ad44f cmake-19.1.5.src.tar.xz +0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5 cmake-v3.31.2.tar.gz +997b493fb604e5e2c5b11c765a4c42b37acf00a4d6e8a14f8108d5c1051d760f libunwind-19.1.5.src.tar.xz +f71835d49461a15c283aa9030a854abfd7c651a685d711a67158644b043f6f14 lld-19.1.5.src.tar.xz +7d71635948e4da1814ce8e15ec45399e4094a5442e86d352c96ded0f2b3171b6 llvm-19.1.5.src.tar.xz +22b352c35b034a4ab3f2b852b6a2602a4da8971abe459080450d9e3462550d1d third-party-19.1.5.src.tar.xz +821bdff48a3f683bc4bb3b6f0b5fe7b2d647cf65d52aeb63328c91a6c6df285a v1.12.1.tar.gz +8c29e06cf42aacc1eafc4077ae2ec6c6fcb96a626157e0593d5e82a34fd403c1 zstd-1.5.6.tar.gz diff --git a/SHA256SUM.manylinux_2_28 b/SHA256SUM.manylinux_2_28 new file mode 100644 index 0000000..136490f --- /dev/null +++ b/SHA256SUM.manylinux_2_28 @@ -0,0 +1,7 @@ +e7dfc8050407b5cc564c1c1afe19517255c9229cccd886dbd5bac9b652828d85 clang-19.1.5.src.tar.xz +a08ae477571fd5e929c27d3d0d28c6168d58dd00b6354c2de3266ae0d86ad44f cmake-19.1.5.src.tar.xz +997b493fb604e5e2c5b11c765a4c42b37acf00a4d6e8a14f8108d5c1051d760f libunwind-19.1.5.src.tar.xz +f71835d49461a15c283aa9030a854abfd7c651a685d711a67158644b043f6f14 lld-19.1.5.src.tar.xz +7d71635948e4da1814ce8e15ec45399e4094a5442e86d352c96ded0f2b3171b6 llvm-19.1.5.src.tar.xz +22b352c35b034a4ab3f2b852b6a2602a4da8971abe459080450d9e3462550d1d third-party-19.1.5.src.tar.xz +821bdff48a3f683bc4bb3b6f0b5fe7b2d647cf65d52aeb63328c91a6c6df285a v1.12.1.tar.gz diff --git a/build-manylinux.sh b/build-manylinux.sh new file mode 100755 index 0000000..81e672d --- /dev/null +++ b/build-manylinux.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors + +git config --global --add safe.directory $(pwd) + +CMAKE_BUILD_TYPE="Release" +IS_BUILD_TARGET=true +IS_NINJA=true +CMAKE_OPTS="" + +for i in "$@"; do + case $i in + --release|--Release) + CMAKE_BUILD_TYPE="Release" + shift + ;; + --debug|--Debug) + CMAKE_BUILD_TYPE="Debug" + shift + ;; + --not-build) + IS_BUILD_TARGET=false + shift + ;; + --not-ninja) + IS_NINJA=false + shift + ;; + *) + CMAKE_OPTS="${CMAKE_OPTS} $i" + shift + ;; + esac +done + +if $IS_NINJA; then + if ! cmake -Bbuild -GNinja -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DWASMEDGE_BUILD_PACKAGE="TGZ;TBZ2;TXZ;TZST;RPM" ${CMAKE_OPTS} .; then + echo === CMakeOutput.log === + cat build/CMakeFiles/CMakeOutput.log + echo === CMakeError.log === + cat build/CMakeFiles/CMakeError.log + exit 1 + fi +else + rm -rf build + mkdir build + cd build + if ! cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DWASMEDGE_BUILD_PACKAGE="TGZ;TBZ2;TXZ;TZST;RPM;DEB" ${CMAKE_OPTS} ..; then + cd .. + echo === CMakeOutput.log === + cat build/CMakeFiles/CMakeOutput.log + echo === CMakeError.log === + cat build/CMakeFiles/CMakeError.log + exit 1 + fi + cd .. +fi + +if ${IS_BUILD_TARGET}; then + cmake --build build + cmake --build build --target package +fi diff --git a/docker-bake.alpine-base.hcl b/docker-bake.alpine-base.hcl new file mode 100644 index 0000000..e64a83b --- /dev/null +++ b/docker-bake.alpine-base.hcl @@ -0,0 +1,12 @@ +group "default" { + targets = ["alpine-base"] +} + +target "alpine-base" { + dockerfile = "Dockerfile.alpine-base" + context = "." + # Build for both major architectures at once + platforms = ["linux/amd64", "linux/arm64"] + # Tag it clearly so we can use it later + tags = ["wasmedge/wasmedge:alpine-base-3.23"] +} diff --git a/docker-bake.alpine-static.hcl b/docker-bake.alpine-static.hcl new file mode 100644 index 0000000..6dbcaf1 --- /dev/null +++ b/docker-bake.alpine-static.hcl @@ -0,0 +1,20 @@ +group "default" { + targets = ["cross"] +} + +target "base" { + dockerfile = "./Dockerfile.alpine-static" + context = "." + contexts = { + source = "./WasmEdge-0.18.1" + } + output = ["build"] +} + +target "cross" { + inherits = ["base"] + platforms = [ + "linux/amd64", + "linux/arm64" + ] +} diff --git a/docker-bake.ci-image-base.hcl b/docker-bake.ci-image-base.hcl new file mode 100644 index 0000000..8512498 --- /dev/null +++ b/docker-bake.ci-image-base.hcl @@ -0,0 +1,26 @@ +group "default" { + targets = [ + "x86_64", + "aarch64" + ] +} + +target "base" { + dockerfile = "Dockerfile.ci-image-base" + context = "." +} + +target "x86_64" { + inherits = ["base"] + platforms = ["linux/amd64"] + tags = [ + "wasmedge/wasmedge:ci-image-base", + "wasmedge/wasmedge:ci-image-base_x86_64" + ] +} + +target "aarch64" { + inherits = ["base"] + platforms = ["linux/arm64"] + tags = ["wasmedge/wasmedge:ci-image-base_aarch64"] +} diff --git a/docker-bake.debian-static.hcl b/docker-bake.debian-static.hcl new file mode 100644 index 0000000..874adf6 --- /dev/null +++ b/docker-bake.debian-static.hcl @@ -0,0 +1,20 @@ +group "default" { + targets = ["cross"] +} + +target "base" { + dockerfile = "./Dockerfile.debian-static" + context = "." + contexts = { + source = "./WasmEdge-0.18.1" + } + output = ["build"] +} + +target "cross" { + inherits = ["base"] + platforms = [ + "linux/amd64", + "linux/arm64" + ] +} diff --git a/docker-bake.manylinux.hcl b/docker-bake.manylinux.hcl index 48e0cbf..a605017 100644 --- a/docker-bake.manylinux.hcl +++ b/docker-bake.manylinux.hcl @@ -4,7 +4,7 @@ target "base" { } target "plugins-base" { - dockerfile = "Dockerfile.manylinux_2_28-plugins-deps" + dockerfile = "./Dockerfile.manylinux_2_28-plugins-deps" context = "." } diff --git a/docker-bake.ubuntu.hcl b/docker-bake.ubuntu.hcl index d856572..c608e89 100644 --- a/docker-bake.ubuntu.hcl +++ b/docker-bake.ubuntu.hcl @@ -1,13 +1,13 @@ group "default" { targets = [ - "latest", + "cuda", + "final" ] } group "latest" { targets = [ "base-2404-clang", - "plugins-2404-clang", ] } @@ -115,7 +115,7 @@ target "base" { } target "plugins" { - dockerfile = "Dockerfile.ubuntu-plugins-deps" + dockerfile = "./Dockerfile.ubuntu-plugins-deps" context = "." matrix = { @@ -133,7 +133,7 @@ target "plugins" { } } -target "all-versions" { +target "final" { matrix = { parent = ["base", "plugins"] ubuntu = ["20.04", "22.04", "24.04"] diff --git a/scripts/ffmpeg/download-ffmpeg-sample-video.sh b/scripts/ffmpeg/download-ffmpeg-sample-video.sh new file mode 100644 index 0000000..4cff55d --- /dev/null +++ b/scripts/ffmpeg/download-ffmpeg-sample-video.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors + +# The video below is sourced from an ffmpeg-libav-tutorial repository. +# Source: https://github.com/leandromoreira/ffmpeg-libav-tutorial/blob/master/LICENSE. +TODIR=$1 +SAMPLE_VIDEO=https://raw.githubusercontent.com/Hrushi20/rust-ffmpeg/master/assets/bunny.mp4 +if [[ $# -eq 0 ]]; then + TODIR=. +fi +if [ ! -d $TODIR ]; then + mkdir $TODIR +fi + +if [ ! -f $TODIR/sample_video.mp4 ]; then + curl -sL $SAMPLE_VIDEO -o $TODIR/sample_video.mp4 + cp $TODIR/sample_video.mp4 $TODIR/dummy.mp4 # Dummy file to manipulate and run tests on file. +fi diff --git a/scripts/ffmpeg/install-ffmpeg-v7.1.1.sh b/scripts/ffmpeg/install-ffmpeg-v7.1.1.sh new file mode 100644 index 0000000..4e04994 --- /dev/null +++ b/scripts/ffmpeg/install-ffmpeg-v7.1.1.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -e + +curl -sL https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n7.1.1.zip -o ffmpeg.zip + +unzip ffmpeg.zip + +mkdir -p FFmpeg-n7.1.1/output +cd FFmpeg-n7.1.1 +./configure --prefix=$(pwd)/output --enable-gpl --enable-nonfree --enable-shared --disable-static +make && make install +cd .. + +rm -rf ffmpeg.zip diff --git a/scripts/opencvmini/install-opencvmini.sh b/scripts/opencvmini/install-opencvmini.sh index 43afef9..ec2da38 100644 --- a/scripts/opencvmini/install-opencvmini.sh +++ b/scripts/opencvmini/install-opencvmini.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# SPDX-FileCopyrightText: 2019-2024 Second State INC +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors OPENCV_VERSION=${OPENCV_VERSION:-4.8.0} wget -O opencv.zip https://github.com/opencv/opencv/archive/refs/tags/${OPENCV_VERSION}.zip diff --git a/scripts/wasi-crypto/build-openssl.sh b/scripts/wasi-crypto/build-openssl.sh index 45d628b..545cc46 100755 --- a/scripts/wasi-crypto/build-openssl.sh +++ b/scripts/wasi-crypto/build-openssl.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# SPDX-FileCopyrightText: 2019-2024 Second State INC +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors echo "Building OpenSSL for wasi-crypto..." # Get OpenSSL source @@ -8,7 +8,7 @@ curl -s -L -O --remote-name-all https://www.openssl.org/source/openssl-1.1.1n.ta echo "40dceb51a4f6a5275bde0e6bf20ef4b91bfc32ed57c0552e2e8e15463372b17a openssl-1.1.1n.tar.gz" | sha256sum -c tar -xf openssl-1.1.1n.tar.gz cd ./openssl-1.1.1n -# OpenSSL configure need newer perl +# Configuring OpenSSL requires newer Perl. curl -s -L -O --remote-name-all https://www.cpan.org/src/5.0/perl-5.34.0.tar.gz tar -xf perl-5.34.0.tar.gz cd perl-5.34.0 diff --git a/scripts/wasi-nn/build-wasinn-ubuntu-openvino.sh b/scripts/wasi-nn/build-wasinn-ubuntu-openvino.sh new file mode 100755 index 0000000..7bcbe37 --- /dev/null +++ b/scripts/wasi-nn/build-wasinn-ubuntu-openvino.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors + +if [[ ! -v "${CMAKE_BUILD_TYPE}" ]]; then + CMAKE_BUILD_TYPE=Release +fi + +ldconfig +git config --global --add safe.directory $(pwd) +if ! cmake -Bbuild -GNinja -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE -DWASMEDGE_BUILD_TESTS=ON -DWASMEDGE_PLUGIN_WASI_NN_BACKEND="OpenVINO" .; then + echo === CMakeOutput.log === + cat build/CMakeFiles/CMakeOutput.log + echo === CMakeError.log === + cat build/CMakeFiles/CMakeError.log + exit 1 +fi +cmake --build build diff --git a/scripts/wasi-nn/install-chattts.sh b/scripts/wasi-nn/install-chattts.sh index f406d46..82edc0f 100644 --- a/scripts/wasi-nn/install-chattts.sh +++ b/scripts/wasi-nn/install-chattts.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: 2019-2024 Second State INC +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors set -e @@ -14,7 +14,12 @@ pip install --upgrade pip # Install PyTorch CPU version to save space pip --python /usr/bin/python3 install --break-system-packages --index-url https://download.pytorch.org/whl/cpu 'torch<=2.6.0' 'torchaudio<=2.6.0' -pip --python /usr/bin/python3 install --break-system-packages chattts==0.2.3 + +# The Debian-provided numpy has no RECORD file, so pip cannot uninstall it +# when resolving a newer version. Shadow it with a pip-managed copy instead. +pip --python /usr/bin/python3 install --break-system-packages --ignore-installed 'numpy<3.0.0' + +pip --python /usr/bin/python3 install --break-system-packages chattts==0.2.4, transformers==4.46.3 # Remove wheel cache pip --python /usr/bin/python3 cache purge diff --git a/scripts/wasi-nn/install-libpiper.sh b/scripts/wasi-nn/install-libpiper.sh new file mode 100755 index 0000000..d2242fd --- /dev/null +++ b/scripts/wasi-nn/install-libpiper.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors + +set -e + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +PATCH_PATH="${SCRIPT_DIR}/libpiper.patch" + +if [ ! -f "${PATCH_PATH}" ]; then + echo "Error: libpiper.patch not found at ${PATCH_PATH}" + exit 1 +fi + +PIPER_REPO="https://github.com/OHF-Voice/piper1-gpl.git" +PIPER_COMMIT="32b95f8c1f0dc0ce27a6acd1143de331f61af777" +PIPER_INSTALL_TO="/usr/local" + +case "$(uname -m)" in + 'x86_64') ;; + 'aarch64') ;; + *) + echo "Unsupported architecture for libpiper: $(uname -m)" >&2 + exit 1 + ;; +esac + +rm -rf piper-source +git clone --depth 1 "${PIPER_REPO}" piper-source +cd piper-source +git fetch --depth 1 origin "${PIPER_COMMIT}" +git checkout FETCH_HEAD + +cp "${PATCH_PATH}" . +patch -p1 < libpiper.patch +cd libpiper + +cmake -Bbuild \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${PIPER_INSTALL_TO}" \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON +cmake --build build --parallel $(nproc) +cmake --install build + +echo "Copying espeak-ng static libraries to ${PIPER_INSTALL_TO}/lib..." + +if [ -f "build/espeak_ng-install/lib/libespeak-ng.a" ]; then + cp "build/espeak_ng-install/lib/libespeak-ng.a" "${PIPER_INSTALL_TO}/lib/" +else + find build -name "libespeak-ng.a" -exec cp {} "${PIPER_INSTALL_TO}/lib/" \; -quit +fi + +find build -name "libucd.a" -exec cp {} "${PIPER_INSTALL_TO}/lib/" \; -quit + +if [ ! -f "${PIPER_INSTALL_TO}/lib/libespeak-ng.a" ]; then + echo "Error: Failed to install libespeak-ng.a" + exit 1 +fi + +if [ ! -f "${PIPER_INSTALL_TO}/lib/libucd.a" ]; then + echo "Error: Failed to install libucd.a" + exit 1 +fi + +echo "Installing espeak-ng-data to ${PIPER_INSTALL_TO}/share..." + +if [ -d "build/espeak_ng-install/share/espeak-ng-data" ]; then + mkdir -p "${PIPER_INSTALL_TO}/share" + cp -r "build/espeak_ng-install/share/espeak-ng-data" "${PIPER_INSTALL_TO}/share/" + echo "Espeak-ng-data installed successfully." +else + echo "Error: espeak-ng-data directory not found in build tree!" + exit 1 +fi + +cd ../.. +rm -rf piper-source + +ldconfig diff --git a/scripts/wasi-nn/install-onnxruntime.sh b/scripts/wasi-nn/install-onnxruntime.sh index dad1fb5..4943833 100644 --- a/scripts/wasi-nn/install-onnxruntime.sh +++ b/scripts/wasi-nn/install-onnxruntime.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: 2019-2024 Second State INC +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors set -e diff --git a/scripts/wasi-nn/install-openvino-genai.sh b/scripts/wasi-nn/install-openvino-genai.sh index 2be3691..a926af3 100644 --- a/scripts/wasi-nn/install-openvino-genai.sh +++ b/scripts/wasi-nn/install-openvino-genai.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# SPDX-FileCopyrightText: 2019-2024 Second State INC +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors set -e if [[ ! -v "${OPENVINOGEN_VERSION}" ]]; then diff --git a/scripts/wasi-nn/install-openvino.sh b/scripts/wasi-nn/install-openvino.sh index 432f1ac..d74b62c 100755 --- a/scripts/wasi-nn/install-openvino.sh +++ b/scripts/wasi-nn/install-openvino.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# SPDX-FileCopyrightText: 2019-2024 Second State INC +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors set -e if [[ ! -v "${OPENVINO_VERSION}" ]]; then diff --git a/scripts/wasi-nn/install-pytorch.sh b/scripts/wasi-nn/install-pytorch.sh index 55f77ac..c3097b4 100755 --- a/scripts/wasi-nn/install-pytorch.sh +++ b/scripts/wasi-nn/install-pytorch.sh @@ -1,31 +1,39 @@ #!/usr/bin/env bash # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# SPDX-FileCopyrightText: 2019-2024 Second State INC +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors +set -e if [[ ! -n ${PYTORCH_VERSION} ]]; then - PYTORCH_VERSION="2.5.1" + PYTORCH_VERSION="2.12.1" fi -if [[ ! -n ${PYTORCH_INSTALL_TO} ]]; then - PYTORCH_INSTALL_TO=. -fi - -PYTORCH_LINK="libtorch-cxx11-abi" -PYTORCH_SHA="618ca54eef82a1dca46ff1993d5807d9c0deb0bae147da4974166a147cb562fa" +PYTORCH_DEFAULT_DIR="libtorch-cxx11abi" +PYTORCH_SHA="78b99d296e0557742c780266a3fb94824ec382e8a285f4d8edf92e033572c5d4" for i in "$@"; do case $i in --disable-cxx11-abi) - PYTORCH_LINK="libtorch" + # The pre-cxx11-abi build is only published up to torch 2.5.1. + # This mode is kept for the branches still building with the pre-cxx11 + # ABI; remove it after the 0.18.0 release. + PYTORCH_VERSION="2.5.1" + PYTORCH_DEFAULT_DIR="libtorch" PYTORCH_SHA="21d05ad61935fc70912c779443dba112bda9c9ec1c999345d724935828f81c55" shift ;; esac done -if [ ! -d ${PYTORCH_INSTALL_TO}/libtorch ]; then - curl -s -L -O --remote-name-all https://download.pytorch.org/libtorch/cpu/${PYTORCH_LINK}-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip - echo "${PYTORCH_SHA} ${PYTORCH_LINK}-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip" | sha256sum -c - unzip -q "${PYTORCH_LINK}-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip" -d ${PYTORCH_INSTALL_TO} - rm -f "${PYTORCH_LINK}-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip" +if [[ ! -n ${PYTORCH_INSTALL_TO} ]]; then + PYTORCH_INSTALL_TO=${PYTORCH_DEFAULT_DIR} +fi + +if [ ! -d ${PYTORCH_INSTALL_TO} ]; then + curl -sSf -L -O --remote-name-all https://download.pytorch.org/libtorch/cpu/libtorch-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip + echo "${PYTORCH_SHA} libtorch-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip" | sha256sum -c + PYTORCH_UNPACK_DIR=$(mktemp -d -p $(dirname ${PYTORCH_INSTALL_TO})) + unzip -q "libtorch-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip" -d ${PYTORCH_UNPACK_DIR} + mv ${PYTORCH_UNPACK_DIR}/libtorch ${PYTORCH_INSTALL_TO} + rmdir ${PYTORCH_UNPACK_DIR} + rm -f "libtorch-shared-with-deps-${PYTORCH_VERSION}%2Bcpu.zip" fi diff --git a/scripts/wasi-nn/libpiper.patch b/scripts/wasi-nn/libpiper.patch new file mode 100644 index 0000000..e74cb4b --- /dev/null +++ b/scripts/wasi-nn/libpiper.patch @@ -0,0 +1,22 @@ +diff --git a/libpiper/CMakeLists.txt b/libpiper/CMakeLists.txt +index 57349c1..2e7ddd8 100644 +--- a/libpiper/CMakeLists.txt ++++ b/libpiper/CMakeLists.txt +@@ -2,6 +2,8 @@ + cmake_minimum_required(VERSION 3.26) + project(piper LANGUAGES C CXX) + ++option(BUILD_SHARED_LIBS "Build using shared libraries" OFF) ++ + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + +@@ -112,7 +114,7 @@ endif() + + # ---- libpiper --- + +-add_library(piper SHARED ++add_library(piper + ${CMAKE_CURRENT_SOURCE_DIR}/src/piper.cpp + ) + diff --git a/scripts/wasi-nn/test-wasinn-ubuntu-openvino.sh b/scripts/wasi-nn/test-wasinn-ubuntu-openvino.sh new file mode 100755 index 0000000..49dd01a --- /dev/null +++ b/scripts/wasi-nn/test-wasinn-ubuntu-openvino.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The WasmEdge Authors + +ldconfig +export LD_LIBRARY_PATH="$(pwd)/build/lib/api:$LD_LIBRARY_PATH" + +cd build +ctest +cd -