From a79d7f541c1d2c1345d71cc7eb454b970479e8f9 Mon Sep 17 00:00:00 2001 From: HwangRock Date: Sat, 3 Oct 2026 17:52:14 +0900 Subject: [PATCH] [ZEPPELIN-6068] Support ARM64 (multi-arch) Docker images - Select architecture-specific parts (JAVA_HOME, Miniconda installer, micromamba) from TARGETARCH so the images build for amd64 and arm64 - Replace the dead micromamba source (snakepit) with micro.mamba.pm - Create the conda env with `mamba env create` (required on mamba 2.x) - Make Z_VERSION a build ARG for the all-in-one image - Add the docker-publish workflow for multi-arch Docker Hub push - Update the docker deployment docs to use buildx --- .github/workflows/docker-publish.yml | 86 +++++++++++++++++++ docs/setup/deployment/docker.md | 34 +++++++- .../docker/zeppelin-interpreter/Dockerfile | 9 +- scripts/docker/zeppelin-server/Dockerfile | 4 +- scripts/docker/zeppelin/bin/Dockerfile | 20 +++-- 5 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 00000000000..1314dcb795c --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: docker-publish + +on: + workflow_dispatch: + inputs: + version: + description: 'Released Zeppelin version to build and tag (e.g. 0.12.0)' + required: true + type: string + push_latest: + description: 'Also push the "latest" tag' + required: false + type: boolean + default: false + dry_run: + description: 'Build only, do not log in to or push to Docker Hub' + required: false + type: boolean + default: true + +permissions: + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Validate version + env: + VERSION: ${{ inputs.version }} + run: | + if [[ -z "${VERSION}" ]]; then + echo "version is required" >&2 + exit 1 + fi + if [[ "${VERSION}" == *SNAPSHOT* ]]; then + echo "SNAPSHOT versions cannot be published: ${VERSION}" >&2 + exit 1 + fi + if [[ ! "${VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "version must look like X.Y.Z: ${VERSION}" >&2 + exit 1 + fi + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Log in to Docker Hub + if: ${{ !inputs.dry_run }} + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Build and push + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: scripts/docker/zeppelin/bin + platforms: linux/amd64,linux/arm64 + build-args: | + Z_VERSION=${{ inputs.version }} + push: ${{ !inputs.dry_run }} + tags: | + apache/zeppelin:${{ inputs.version }} + ${{ inputs.push_latest && 'apache/zeppelin:latest' || '' }} diff --git a/docs/setup/deployment/docker.md b/docs/setup/deployment/docker.md index 7e112db6c47..4ce49085c22 100644 --- a/docs/setup/deployment/docker.md +++ b/docs/setup/deployment/docker.md @@ -62,34 +62,60 @@ docker run -p 8080:8080 --rm \ ### Building dockerfile locally +The Dockerfiles select architecture-specific packages through the `TARGETARCH` build argument, which is set automatically by `docker buildx` (BuildKit). +So **`docker buildx` is required**: the legacy `docker build` without BuildKit does not set `TARGETARCH` and the build fails. + +To build an image for both amd64 and arm64 (e.g. Apple Silicon): + ```bash cd $ZEPPELIN_HOME cd scripts/docker/zeppelin/bin -docker build -t my-zeppelin:my-tag ./ +docker buildx build --platform linux/amd64,linux/arm64 \ + --build-arg Z_VERSION= \ + -t my-zeppelin:my-tag --push ./ ``` +A multi-platform image can't be loaded into the local docker image store, so push it to a registry (`--push`) as above. +To build for a single architecture and use it locally, give only one platform and `--load` the result: + +```bash +docker buildx build --platform linux/arm64 \ + --build-arg Z_VERSION= \ + -t my-zeppelin:my-tag --load ./ +``` + +`Z_VERSION` is optional; when it is omitted, the version declared in the Dockerfile is used. + +The official multi-arch `apache/zeppelin` image is built and pushed to Docker Hub by the manually triggered `docker-publish` GitHub Actions workflow (`.github/workflows/docker-publish.yml`). + ### Build docker image for Zeppelin server & interpreters Starting from 0.9, Zeppelin support to run in k8s or docker. So we add the capability to build docker images for Zeppelin server & interpreter. Recommendation: Edit the Docker files yourself to adapt them to your needs and reduce the image size. +These images also require `docker buildx` (see "Building dockerfile locally" above). At first your need to build a zeppelin-distribution docker image. ```bash cd $ZEPPELIN_HOME -docker build -t zeppelin-distribution . +docker buildx build --platform linux/arm64 -t zeppelin-distribution --load . ``` Build docker image for zeppelin server. ```bash cd $ZEPPELIN_HOME/scripts/docker/zeppelin-server -docker build -t zeppelin-server . +docker buildx build --platform linux/arm64 -t zeppelin-server --load . ``` Build base docker image for zeppelin interpreter. ```bash cd $ZEPPELIN_HOME/scripts/docker/zeppelin-interpreter -docker build -t zeppelin-interpreter-base . +docker buildx build --platform linux/arm64 -t zeppelin-interpreter-base --load . ``` +Use `--platform linux/amd64` instead on an amd64 host. The zeppelin-server and zeppelin-interpreter images start `FROM` the +distribution image (`ZEPPELIN_DISTRIBUTION_IMAGE` build argument, default `zeppelin-distribution:latest`), so build them for the same platform as the distribution image. +To publish multi-arch images (`--platform linux/amd64,linux/arm64 --push`), push the distribution image to a registry first and +pass it with `--build-arg ZEPPELIN_DISTRIBUTION_IMAGE=/zeppelin-distribution:`. + diff --git a/scripts/docker/zeppelin-interpreter/Dockerfile b/scripts/docker/zeppelin-interpreter/Dockerfile index 63dfb352dab..d3d7acec7d6 100644 --- a/scripts/docker/zeppelin-interpreter/Dockerfile +++ b/scripts/docker/zeppelin-interpreter/Dockerfile @@ -21,6 +21,8 @@ FROM ubuntu:22.04 LABEL maintainer="Apache Software Foundation " ARG version="0.13.0-SNAPSHOT" +# Set automatically by buildx (amd64 or arm64) +ARG TARGETARCH ENV VERSION="${version}" \ ZEPPELIN_HOME="/opt/zeppelin" @@ -28,9 +30,14 @@ ENV VERSION="${version}" \ # Install Java for zeppelin interpreter # Install micromamba to install a python environment via conda RUN set -ex && \ + case "${TARGETARCH}" in \ + amd64) mamba_arch="linux-64" ;; \ + arm64) mamba_arch="linux-aarch64" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ /usr/bin/apt-get update && \ DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install -y openjdk-11-jre-headless wget tini bzip2 && \ - /usr/bin/wget --remote-encoding=utf-8 -qO- https://micromamba.snakepit.net/api/micromamba/linux-64/latest | tar -xvj bin/micromamba && \ + /usr/bin/wget --remote-encoding=utf-8 -qO- https://micro.mamba.pm/api/micromamba/${mamba_arch}/latest | tar -xvj bin/micromamba && \ # Cleanup /usr/bin/apt-get clean && \ /bin/rm -rf /var/lib/apt/lists/* diff --git a/scripts/docker/zeppelin-server/Dockerfile b/scripts/docker/zeppelin-server/Dockerfile index 232e1e47aee..d6129b45519 100644 --- a/scripts/docker/zeppelin-server/Dockerfile +++ b/scripts/docker/zeppelin-server/Dockerfile @@ -37,10 +37,12 @@ RUN set -ex && \ apt-get clean ARG version="0.13.0-SNAPSHOT" +# Set automatically by buildx (amd64 or arm64), matches the openjdk install path suffix +ARG TARGETARCH ENV LANG=en_US.UTF-8 \ LC_ALL=en_US.UTF-8 \ - JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 \ + JAVA_HOME=/usr/lib/jvm/java-11-openjdk-${TARGETARCH} \ VERSION="${version}" \ HOME="/opt/zeppelin" \ ZEPPELIN_HOME="/opt/zeppelin" \ diff --git a/scripts/docker/zeppelin/bin/Dockerfile b/scripts/docker/zeppelin/bin/Dockerfile index a7ffae2b94f..8c3c0736e7b 100644 --- a/scripts/docker/zeppelin/bin/Dockerfile +++ b/scripts/docker/zeppelin/bin/Dockerfile @@ -17,14 +17,18 @@ FROM ubuntu:20.04 LABEL maintainer="Apache Software Foundation " -ENV Z_VERSION="0.13.0-SNAPSHOT" +ARG Z_VERSION="0.13.0-SNAPSHOT" +ENV Z_VERSION=${Z_VERSION} + +# Set automatically by buildx (amd64 or arm64), matches the openjdk install path suffix +ARG TARGETARCH ENV LOG_TAG="[ZEPPELIN_${Z_VERSION}]:" \ ZEPPELIN_HOME="/opt/zeppelin" \ HOME="/opt/zeppelin" \ LANG=en_US.UTF-8 \ LC_ALL=en_US.UTF-8 \ - JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64 \ + JAVA_HOME=/usr/lib/jvm/java-11-openjdk-${TARGETARCH} \ ZEPPELIN_ADDR="0.0.0.0" RUN echo "$LOG_TAG install basic packages" && \ @@ -39,12 +43,18 @@ RUN echo "$LOG_TAG install basic packages" && \ # Install conda to manage python packages ARG miniconda_version="py39_24.1.2-0" # Hashes via https://docs.conda.io/en/latest/miniconda_hashes.html -ARG miniconda_sha256="2ec135e4ae2154bb41e8df9ecac7ef23a7d6ca59fc1c8071cfe5298505c19140" +ARG miniconda_sha256_amd64="2ec135e4ae2154bb41e8df9ecac7ef23a7d6ca59fc1c8071cfe5298505c19140" +ARG miniconda_sha256_arm64="b3e7d8ad4a4c9106594b268ab1cd9494ce982eaf7734bb2cd13a47e14e92a43e" # Install python packages via conda COPY env_python_3.yml /env_python_3.yml RUN set -ex && \ - wget -nv https://repo.anaconda.com/miniconda/Miniconda3-${miniconda_version}-Linux-x86_64.sh -O miniconda.sh && \ + case "${TARGETARCH}" in \ + amd64) miniconda_arch="x86_64"; miniconda_sha256="${miniconda_sha256_amd64}" ;; \ + arm64) miniconda_arch="aarch64"; miniconda_sha256="${miniconda_sha256_arm64}" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \ + esac && \ + wget -nv https://repo.anaconda.com/miniconda/Miniconda3-${miniconda_version}-Linux-${miniconda_arch}.sh -O miniconda.sh && \ echo "${miniconda_sha256} miniconda.sh" > anaconda.sha256 && \ sha256sum --strict -c anaconda.sha256 && \ bash miniconda.sh -b -p /opt/conda && \ @@ -52,7 +62,7 @@ RUN set -ex && \ conda config --set always_yes yes --set changeps1 no && \ conda info -a && \ conda install mamba -c conda-forge && \ - mamba env update -f /env_python_3.yml --prune && \ + mamba env create -f /env_python_3.yml && \ # Cleanup rm -v miniconda.sh anaconda.sha256 && \ # Cleanup based on https://github.com/ContinuumIO/docker-images/commit/cac3352bf21a26fa0b97925b578fb24a0fe8c383