diff --git a/.github/workflows/release-build.yml b/.github/workflows/release-build.yml index ab2f0e9abc22..c0564c3b105e 100644 --- a/.github/workflows/release-build.yml +++ b/.github/workflows/release-build.yml @@ -13,30 +13,52 @@ on: jobs: releases: name: Releases - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.runner }} strategy: fail-fast: false matrix: - target: - - 'bin-Fedora' - - 'bin-Ubuntu-jammy' - - 'bin-Ubuntu-noble' - - 'bin-Ubuntu-resolute' + # dir is where build-release.sh puts the target's output; arch picks + # the merged artifact, since each architecture is checksummed and + # signed separately. + include: + - { target: 'bin-Fedora', runner: 'ubuntu-24.04', arch: 'amd64', dir: 'release' } + - { target: 'bin-Ubuntu-jammy', runner: 'ubuntu-24.04', arch: 'amd64', dir: 'release' } + - { target: 'bin-Ubuntu-noble', runner: 'ubuntu-24.04', arch: 'amd64', dir: 'release' } + - { target: 'bin-Ubuntu-resolute', runner: 'ubuntu-24.04', arch: 'amd64', dir: 'release' } + # arm64 builds natively on an arm64 runner. + - { target: 'bin-Ubuntu-jammy-arm64', runner: 'ubuntu-24.04-arm', arch: 'arm64', dir: 'release-arm64' } + - { target: 'bin-Ubuntu-noble-arm64', runner: 'ubuntu-24.04-arm', arch: 'arm64', dir: 'release-arm64' } + - { target: 'bin-Ubuntu-resolute-arm64', runner: 'ubuntu-24.04-arm', arch: 'arm64', dir: 'release-arm64' } + # armv7 cross-compiles in the amd64 builder image (see + # tools/repro-build.armv7.sh), with qemu for the few armv7 programs + # the build runs. + - { target: 'bin-Ubuntu-jammy-armv7', runner: 'ubuntu-24.04', arch: 'armv7', dir: 'release-armv7' } + - { target: 'bin-Ubuntu-noble-armv7', runner: 'ubuntu-24.04', arch: 'armv7', dir: 'release-armv7' } + - { target: 'bin-Ubuntu-resolute-armv7', runner: 'ubuntu-24.04', arch: 'armv7', dir: 'release-armv7' } steps: - name: Git checkout uses: actions/checkout@v6 with: fetch-depth: 0 + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + with: + platforms: arm + if: endsWith(matrix.target, '-armv7') + - name: Build environment setup run: | distribution=$(echo ${{ matrix.target }} | cut -d'-' -f3) + # Same naming as contrib/cl-repro.sh: arm64 images are suffixed. + suffix="" + [ "$(dpkg --print-architecture)" = arm64 ] && suffix="-arm64" sudo docker run --rm -v $(pwd):/build ubuntu:${distribution} bash -c "\ apt-get update && \ apt-get install -y debootstrap && \ debootstrap ${distribution} /build/${distribution}" sudo tar -C ${distribution} -c . | docker import - ${distribution} - docker build -t cl-repro-${distribution} - < contrib/reprobuild/Dockerfile.${distribution} + docker build -t cl-repro-${distribution}${suffix} - < contrib/reprobuild/Dockerfile.${distribution} if: contains(matrix.target, 'Ubuntu') - name: Build release @@ -53,8 +75,8 @@ jobs: - name: Upload target artifacts uses: actions/upload-artifact@v7 with: - path: release/ - name: ${{ matrix.target }} + path: ${{ matrix.dir }}/ + name: ${{ matrix.arch }}-${{ matrix.target }} if-no-files-found: error artifact: @@ -62,9 +84,23 @@ jobs: needs: releases runs-on: ubuntu-24.04 steps: - - name: Merge artifacts + - name: Merge amd64 artifacts uses: actions/upload-artifact/merge@v7 with: name: c-lightning-${{ inputs.version }} - pattern: bin-* + pattern: amd64-bin-* + delete-merged: true + + - name: Merge arm64 artifacts + uses: actions/upload-artifact/merge@v7 + with: + name: c-lightning-${{ inputs.version }}-arm64 + pattern: arm64-bin-* + delete-merged: true + + - name: Merge armv7 artifacts + uses: actions/upload-artifact/merge@v7 + with: + name: c-lightning-${{ inputs.version }}-armv7 + pattern: armv7-bin-* delete-merged: true diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index 39a33324160c..10b538886026 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -26,6 +26,18 @@ jobs: name: c-lightning-${{ inputs.version }} path: release/ + - name: Download arm64 artifact + uses: actions/download-artifact@v8 + with: + name: c-lightning-${{ inputs.version }}-arm64 + path: release-arm64/ + + - name: Download armv7 artifact + uses: actions/download-artifact@v8 + with: + name: c-lightning-${{ inputs.version }}-armv7 + path: release-armv7/ + - name: Import GPG keys id: gpg uses: crazy-max/ghaction-import-gpg@v7 @@ -40,7 +52,15 @@ jobs: run: echo "default-key $GPG_KEYID" >> ~/.gnupg/gpg.conf - name: Sign release - run: tools/build-release.sh --without-zip sign + env: + INPUT_VERSION: ${{ inputs.version }} + run: | + tools/build-release.sh --without-zip sign + for arch in arm64 armv7; do + tools/sign-release-arm.sh "$INPUT_VERSION" "$arch" + # Their manifests and signatures sit next to the amd64 ones. + mv release-"$arch"/SHA256SUMS-* release/ + done - name: Upload signed artifact uses: actions/upload-artifact@v7 @@ -49,6 +69,20 @@ jobs: overwrite: true path: release/ + - name: Upload arm64 artifact + uses: actions/upload-artifact@v7 + with: + name: c-lightning-${{ inputs.version }}-arm64 + overwrite: true + path: release-arm64/ + + - name: Upload armv7 artifact + uses: actions/upload-artifact@v7 + with: + name: c-lightning-${{ inputs.version }}-armv7 + overwrite: true + path: release-armv7/ + - name: Determine release data id: release_data env: @@ -60,13 +94,24 @@ jobs: RELEASE_TITLE=$(echo $CHANGELOG_TITLE | cut -d'"' -f2) echo "release_title=$RELEASE_TITLE" >> "$GITHUB_OUTPUT" + # Never replace what is already on the release: release captains add + # their signatures to the .asc files by hand, and a re-run must keep + # them. tools/publish-release-assets.sh uploads only missing files, + # refuses any that differ from the published copy, and appends our + # signature to an existing .asc instead of overwriting it. - name: Prepare release draft if: ${{ github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.create_release) }} - uses: softprops/action-gh-release@v3 - with: - name: "${{ inputs.version }} ${{ steps.release_data.outputs.release_title }}" - tag_name: ${{ inputs.version }} - draft: true - prerelease: contains(inputs.version, "-rc") - files: release/* - fail_on_unmatched_files: true + env: + GH_TOKEN: ${{ github.token }} + INPUT_VERSION: ${{ inputs.version }} + RELEASE_TITLE: ${{ steps.release_data.outputs.release_title }} + run: | + if ! gh release view "$INPUT_VERSION" >/dev/null 2>&1; then + PRERELEASE="" + case "$INPUT_VERSION" in *-rc*) PRERELEASE=--prerelease;; esac + # --target only matters for an untagged test run: gh then + # creates the tag, as the previous upload action did. + gh release create "$INPUT_VERSION" --draft --target "$GITHUB_SHA" $PRERELEASE \ + --title "$INPUT_VERSION $RELEASE_TITLE" --notes "" + fi + tools/publish-release-assets.sh "$INPUT_VERSION" release/* release-arm64/* release-armv7/* diff --git a/.gitignore b/.gitignore index 2a860dd6aaeb..c38c89398d68 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,8 @@ jammy/ noble/ resolute/ release/ +release-arm64/ +release-armv7/ .vscode/ .cache/ diff --git a/cln-rpc/Makefile b/cln-rpc/Makefile index 4bd6b832569a..2351211e3ce1 100644 --- a/cln-rpc/Makefile +++ b/cln-rpc/Makefile @@ -1,14 +1,14 @@ cln-rpc-wrongdir: $(MAKE) -C .. cln-rpc-all -CLN_RPC_EXAMPLES := target/${RUST_PROFILE}/examples/cln-rpc-getinfo +CLN_RPC_EXAMPLES := $(RUST_TARGET_DIR)/examples/cln-rpc-getinfo CLN_RPC_GENALL = cln-rpc/src/model.rs cln-rpc/src/notifications.rs cln-rpc/src/hooks.rs CLN_RPC_SOURCES = $(shell find cln-rpc -name *.rs) ${CLN_RPC_GENALL} DEFAULT_TARGETS += $(CLN_RPC_EXAMPLES) $(CLN_RPC_GENALL) MSGGEN_GENALL += $(CLN_RPC_GENALL) -target/${RUST_PROFILE}/examples/cln-rpc-getinfo: ${CLN_RPC_SOURCES} cln-rpc/examples/getinfo.rs +$(RUST_TARGET_DIR)/examples/cln-rpc-getinfo: ${CLN_RPC_SOURCES} cln-rpc/examples/getinfo.rs $(CARGO) build ${CARGO_OPTS} --example cln-rpc-getinfo cln-rpc-all: ${CLN_RPC_GENALL} ${CLN_RPC_EXAMPLES} diff --git a/contrib/cl-repro.sh b/contrib/cl-repro.sh index e16d6b95341d..5e8ee36930c0 100755 --- a/contrib/cl-repro.sh +++ b/contrib/cl-repro.sh @@ -22,6 +22,10 @@ for v in jammy noble resolute; do echo "$v release:" sudo docker run ubuntu:$v cat /etc/lsb-release echo "Building CL repro $v:" + # arm64 images are suffixed so both architectures can coexist, and so the + # bin-Ubuntu--arm64 targets resolve. amd64 keeps its bare name. + SUFFIX="" + [ "$(dpkg --print-architecture)" = arm64 ] && SUFFIX="-arm64" # shellcheck disable=SC2024 - sudo docker build --no-cache -t cl-repro-$v - < "$LIGHTNING_DIR"/contrib/reprobuild/Dockerfile.$v + sudo docker build --no-cache -t cl-repro-$v$SUFFIX - < "$LIGHTNING_DIR"/contrib/reprobuild/Dockerfile.$v done diff --git a/contrib/reprobuild/Dockerfile.jammy b/contrib/reprobuild/Dockerfile.jammy index 398afec016fb..e8c1c8a6356e 100644 --- a/contrib/reprobuild/Dockerfile.jammy +++ b/contrib/reprobuild/Dockerfile.jammy @@ -73,5 +73,15 @@ WORKDIR /build # that we no longer take the zipfile. CMD git clone /repo . \ && uv sync --all-extras --all-groups \ - && uv run tools/repro-build.sh \ - && cp *.xz /repo/release/ + && if [ "$REPRO_ARCH" = armv7 ]; then \ + uv run tools/repro-build.armv7.sh \ + && mkdir -p /repo/release-armv7 \ + && cp *.xz /repo/release-armv7/; \ + elif [ "$(dpkg --print-architecture)" = arm64 ]; then \ + uv run tools/repro-build.arm64.sh \ + && mkdir -p /repo/release-arm64 \ + && cp *.xz /repo/release-arm64/; \ + else \ + uv run tools/repro-build.sh \ + && cp *.xz /repo/release/; \ + fi diff --git a/contrib/reprobuild/Dockerfile.noble b/contrib/reprobuild/Dockerfile.noble index f711c0fc733a..f7a1d3a4b55f 100644 --- a/contrib/reprobuild/Dockerfile.noble +++ b/contrib/reprobuild/Dockerfile.noble @@ -63,5 +63,15 @@ WORKDIR /build # that we no longer take the zipfile. CMD git clone /repo . \ && uv sync --all-extras --all-groups \ - && uv run tools/repro-build.sh \ - && cp *.xz /repo/release/ + && if [ "$REPRO_ARCH" = armv7 ]; then \ + uv run tools/repro-build.armv7.sh \ + && mkdir -p /repo/release-armv7 \ + && cp *.xz /repo/release-armv7/; \ + elif [ "$(dpkg --print-architecture)" = arm64 ]; then \ + uv run tools/repro-build.arm64.sh \ + && mkdir -p /repo/release-arm64 \ + && cp *.xz /repo/release-arm64/; \ + else \ + uv run tools/repro-build.sh \ + && cp *.xz /repo/release/; \ + fi diff --git a/contrib/reprobuild/Dockerfile.resolute b/contrib/reprobuild/Dockerfile.resolute index f72414b908d6..5b783dd7fa57 100644 --- a/contrib/reprobuild/Dockerfile.resolute +++ b/contrib/reprobuild/Dockerfile.resolute @@ -63,5 +63,15 @@ WORKDIR /build # that we no longer take the zipfile. CMD git clone /repo . \ && uv sync --all-extras --all-groups \ - && uv run tools/repro-build.sh \ - && cp *.xz /repo/release/ + && if [ "$REPRO_ARCH" = armv7 ]; then \ + uv run tools/repro-build.armv7.sh \ + && mkdir -p /repo/release-armv7 \ + && cp *.xz /repo/release-armv7/; \ + elif [ "$(dpkg --print-architecture)" = arm64 ]; then \ + uv run tools/repro-build.arm64.sh \ + && mkdir -p /repo/release-arm64 \ + && cp *.xz /repo/release-arm64/; \ + else \ + uv run tools/repro-build.sh \ + && cp *.xz /repo/release/; \ + fi diff --git a/doc/contribute-to-core-lightning/release-checklist.md b/doc/contribute-to-core-lightning/release-checklist.md index 038588d68c9b..6ea1644f65a6 100644 --- a/doc/contribute-to-core-lightning/release-checklist.md +++ b/doc/contribute-to-core-lightning/release-checklist.md @@ -33,7 +33,7 @@ Here's a checklist for the release process. 3. Confirm that the tag will show up for builds with `git describe`. We don't push it to GitHub yet, just in case the following steps fail, and more fixes are required! 4. Run `contrib/cl-repro.sh` to generate the required `cl-repro-` builder images for the reproducible build environment. 5. Execute `tools/build-release.sh bin-Fedora bin-Ubuntu sign` to locally reproduce the release, generating a matching `SHA256SUMS-v` file and signing it with your GPG key. -6. Push the tag to trigger the "Release 🚀" CI action, which drafts a new `vrc1` pre-release on GitHub and uploads reproducible builds alongside the `SHA256SUMS-v` file and its signature from the `cln@blockstream.com` key. +6. Push the tag to trigger the "Release 🚀" CI action, which drafts a new `vrc1` pre-release on GitHub and uploads reproducible builds alongside the `SHA256SUMS-v` file and its signature from the `cln@blockstream.com` key. The arm64 and armv7 tarballs each come with their own manifest, `SHA256SUMS-v-arm64` and `SHA256SUMS-v-armv7`, and its `.asc`. The CI never replaces a file already on the release: re-running it only adds missing files, and appends its signature to an `.asc` that already has yours. 7. Verify your local `SHA256SUMS-v` file matches the one in the draft release, then append your local signatures to the release's `SHA256SUMS-v.asc` file to attest to the build's integrity. 8. Announce rc1 release on core-lightning's release-chat channel on Discord & Telegram. 9. Use `devtools/credit --markdown v` to generate a single contributor list for the release notes. Use `devtools/credit --verbose v` for namer selection and detailed annotations. @@ -119,7 +119,7 @@ Here's a checklist for the release process. 5. Create a new commit that includes the updates from `update-versions` and `CHANGELOG.md`. 6. Tag the release with `git pull && git tag -s v.`. You will be prompted to enter a tag message, ensure this is filled out. 7. Confirm that the tag is properly set up for builds by running `git describe`. -8. Trigger the pre-release by pushing the version tag with `git push origin v.`; the CI will handle drafting the release and uploading the initial signed checksums. +8. Trigger the pre-release by pushing the version tag with `git push origin v.`; the CI will handle drafting the release and uploading the initial signed checksums, for amd64 (`SHA256SUMS-v`), arm64 (`SHA256SUMS-v-arm64`) and armv7 (`SHA256SUMS-v-armv7`). 9. Generate the required builder images by running `contrib/cl-repro.sh`. 10. Sign the release locally by running `tools/build-release.sh bin-Fedora bin-Ubuntu sign` which will sign the release contents and create `SHA256SUMS-v` and `SHA256SUMS-v.asc` in the release folder. 11. Validate that your local checksums `SHA256SUMS-v` match the Draft release's, then add your signatures to the draft release's signature `SHA256SUMS-v.asc` file. diff --git a/doc/getting-started/advanced-setup/repro.md b/doc/getting-started/advanced-setup/repro.md index a95ce0922300..23922981f8f2 100644 --- a/doc/getting-started/advanced-setup/repro.md +++ b/doc/getting-started/advanced-setup/repro.md @@ -116,6 +116,22 @@ ee83cf4948228ab1f644dbd9d28541fd8ef7c453a3fec90462b08371a8686df8 /repo/release/ Repeat this step for each distribution and each architecture you wish to sign. Once all the binaries are in the `release/` subdirectory we can sign the hashes. +## ARM builds (arm64 and armv7) + +Each ARM architecture has its own manifest and signatures: `SHA256SUMS-v-arm64` / `.asc` and `SHA256SUMS-v-armv7` / `.asc`. They are separate because they are reproduced differently (arm64 natively, armv7 cross-compiled), so you can verify and co-sign one without the other. Their output goes to `release-arm64/` and `release-armv7/`, not `release/`, so the manifests never mix. + +- **arm64** builds natively, on an arm64 machine. Running `contrib/cl-repro.sh` there creates the builder images as `cl-repro--arm64`, and `tools/build-release.sh bin-Ubuntu--arm64` (or `docker run --rm -v $(pwd):/repo -ti cl-repro--arm64`) builds the tarball using the pinned packages in `tools/repro-build.arm64.sh`. +- **armv7** is cross-compiled on an amd64 machine, in the same `cl-repro-` image as the amd64 build, using the pinned toolchain and armhf libraries in `tools/repro-build.armv7.sh`. The build runs a few armv7 programs, so register qemu with the kernel first (once per boot): + + ```shell + docker run --privileged --rm tonistiigi/binfmt --install arm + tools/build-release.sh bin-Ubuntu-noble-armv7 + # or directly: + docker run --rm -v $(pwd):/repo -e REPRO_ARCH=armv7 -ti cl-repro-noble + ``` + +Once the tarballs are built, `tools/sign-release-arm.sh v arm64` (or `armv7`) creates and signs `SHA256SUMS-v-arm64` (or `-armv7`). + # Signing the release manifest The release captain is in charge of creating the manifest, whereas contributors and interested bystanders may contribute their signatures to further increase trust in the binaries. diff --git a/tools/build-release.sh b/tools/build-release.sh index 5d18c8105c49..b9ddac8a54f6 100755 --- a/tools/build-release.sh +++ b/tools/build-release.sh @@ -144,7 +144,23 @@ fi TARGETS=${TARGETS:-$ALL_TARGETS} -RELEASEDIR="$(pwd)/release" +# ARM targets get their own release directory per architecture +# (release-arm64/, release-armv7/), and never build the zip: the zip is +# architecture-independent, and its block unconditionally removes release/'s +# copy before rebuilding it from the current HEAD. +case "$TARGETS" in + *-arm64*) + RELEASEDIR="$(pwd)/release-arm64" + WITHOUT_ZIP=true + ;; + *-armv7*) + RELEASEDIR="$(pwd)/release-armv7" + WITHOUT_ZIP=true + ;; + *) + RELEASEDIR="$(pwd)/release" + ;; +esac BARE_VERSION="$(echo "${VERSION}" | sed 's/^v//g')" TARBALL="${RELEASEDIR}/lightningd_${BARE_VERSION}.orig.tar.bz2" DATE=$(date +%Y%m%d%H%M%S) @@ -203,7 +219,17 @@ for target in $TARGETS; do # Capitalize the first letter of distro D=$(echo "$d" | awk '{print toupper(substr($0,1,1))substr($0,2)}') echo "Building Ubuntu $D Image" - docker run --rm -v "$(pwd)":/repo -e FORCE_MTIME="$MTIME" -e FORCE_VERSION="$VERSION" -e MAKEPAR="$MAKEPAR" cl-repro-"$d" + # armv7 is cross-compiled in the amd64 builder image; arm64 + # builds natively in its own cl-repro--arm64 image. + IMAGE=cl-repro-"$d" + REPRO_ARCH="" + case "$d" in + *-armv7) + IMAGE=cl-repro-"${d%-armv7}" + REPRO_ARCH=armv7 + ;; + esac + docker run --rm -v "$(pwd)":/repo -e FORCE_MTIME="$MTIME" -e FORCE_VERSION="$VERSION" -e MAKEPAR="$MAKEPAR" -e REPRO_ARCH="$REPRO_ARCH" "$IMAGE" echo "Ubuntu $D Image Built" done ;; diff --git a/tools/publish-release-assets.sh b/tools/publish-release-assets.sh new file mode 100755 index 000000000000..b5518104609f --- /dev/null +++ b/tools/publish-release-assets.sh @@ -0,0 +1,124 @@ +#! /bin/sh +# Upload release files to an existing GitHub release without replacing +# anything already published there. +# +# - A file the release does not have yet is uploaded. +# - A file the release already has must be byte-identical, or we stop: a +# different tarball or SHA256SUMS means the builds disagree, and nothing is +# touched. +# - A detached signature (.asc) the release already has is merged instead: +# our signature is appended to the published one, so the signatures that +# release captains added by hand are kept. If the published file already +# carries our signature it is left alone, so re-running is harmless. +# +# Usage: tools/publish-release-assets.sh ... +# Needs gh (with GH_TOKEN) and gpg with the signing key that made our .asc. +set -e + +TAG=${1:-} +if [ -z "$TAG" ] || [ $# -lt 2 ]; then + echo "Usage: $0 ..." >&2 + exit 1 +fi +shift + +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT + +gh release view "$TAG" --json assets --jq '.assets[].name' > "$WORK/published" + +is_published() +{ + grep -qxF "$1" "$WORK/published" +} + +fetch() +{ + rm -f "$WORK/remote" + gh release download "$TAG" --pattern "$1" --output "$WORK/remote" +} + +# The fingerprint of every good signature in $1 over $2, one per line. +good_sigs() +{ + gpg --status-fd 1 --verify "$1" "$2" 2>/dev/null | awk '$2 == "VALIDSIG" {print $3}' || true +} + +# Count signature packets, whether or not we have the signer's key. +count_sigs() +{ + gpg --list-packets "$1" 2>/dev/null | grep -c '^:signature packet' || true +} + +# Concatenating armored signatures can leave a block gpg only half-reads +# (no blank line after the BEGIN line), and it then silently skips it. +normalise() +{ + awk '/^-----BEGIN PGP SIGNATURE-----$/{print; getline l; if (l != "") print ""; print l; next} {print}' "$1" +} + +# Pass 1: everything except signatures. Stop before any upload if a +# published file differs from ours. +for f; do + case "$f" in *.asc) continue;; esac + name=$(basename "$f") + if is_published "$name"; then + fetch "$name" + if ! cmp -s "$f" "$WORK/remote"; then + echo "$name: published copy differs from ours, refusing to continue" >&2 + exit 1 + fi + fi +done + +for f; do + case "$f" in *.asc) continue;; esac + name=$(basename "$f") + if is_published "$name"; then + echo "$name: already published, identical" + else + echo "$name: uploading" + gh release upload "$TAG" "$f" + fi +done + +# Pass 2: signatures. The file each one signs is now the published one. +for f; do + case "$f" in *.asc) ;; *) continue;; esac + name=$(basename "$f") + signed="${f%.asc}" + [ -f "$signed" ] || { echo "$name: no $signed next to it" >&2; exit 1; } + + ours=$(good_sigs "$f" "$signed") + [ -n "$ours" ] || { echo "$name: our own signature does not verify" >&2; exit 1; } + + if ! is_published "$name"; then + echo "$name: uploading" + gh release upload "$TAG" "$f" + continue + fi + + fetch "$name" + if good_sigs "$WORK/remote" "$signed" | grep -qxF "$ours"; then + echo "$name: already carries our signature, left alone" + continue + fi + + # Count after normalising: that also repairs a block the published file + # already had that gpg could not read. + normalise "$WORK/remote" > "$WORK/remote.norm" + before=$(count_sigs "$WORK/remote.norm") + mkdir -p "$WORK/merged" + { cat "$WORK/remote.norm"; normalise "$f"; } > "$WORK/merged/$name" + after=$(count_sigs "$WORK/merged/$name") + if [ "$after" -ne $((before + $(count_sigs "$f"))) ]; then + echo "$name: merged file has $after signatures, expected $before plus ours; not uploading" >&2 + exit 1 + fi + good_sigs "$WORK/merged/$name" "$signed" | grep -qxF "$ours" || { + echo "$name: our signature does not verify in the merged file; not uploading" >&2 + exit 1 + } + echo "$name: appending our signature to the $before already published" + gh release upload --clobber "$TAG" "$WORK/merged/$name" +done diff --git a/tools/repro-build.arm64.sh b/tools/repro-build.arm64.sh new file mode 100755 index 000000000000..fa43fcff6e49 --- /dev/null +++ b/tools/repro-build.arm64.sh @@ -0,0 +1,164 @@ +#! /bin/sh +# ARM64 variant of repro-build.sh. +# Identical to that script except the pinned .deb set: same packages at the +# same upstream versions, built for arm64. Kept separate so the amd64 path +# cannot be affected by arm64 work. + +set -e + +LANG=C +LC_ALL=C +export LANG LC_ALL + +for arg; do + case "$arg" in + --force-version=*) + FORCE_VERSION=${arg#*=} + ;; + --force-mtime=*) + FORCE_MTIME=${arg#*=} + ;; + --help) + echo "Usage: [--force-version=] [--force-mtime=YYYY-MM-DD]" + exit 0 + ;; + *) + echo "Unknown arg $arg" >&2 + exit 1 + ;; + esac + shift +done + +# Taken from https://unix.stackexchange.com/questions/6345/how-can-i-get-distribution-name-and-version-number-in-a-simple-shell-script +if [ -f /etc/os-release ]; then + # freedesktop.org and systemd + # shellcheck disable=SC1091 + . /etc/os-release + OS=$NAME + VER=$VERSION_ID +elif command -v lsb_release >/dev/null 2>&1; then + # linuxbase.org + OS=$(lsb_release -si) + VER=$(lsb_release -sr) +elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + # shellcheck disable=SC1091 + . /etc/lsb-release + OS=$DISTRIB_ID + VER=$DISTRIB_RELEASE +elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + OS=Debian + VER=$(cat /etc/debian_version) +else + # Fall back to uname, e.g. "Linux ", also works for BSD, etc. + OS=$(uname -s) + VER=$(uname -r) +fi + +ARCH=$(dpkg --print-architecture) +PLATFORM="$OS"-"$VER" +VERSION=${FORCE_VERSION:-$(git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's,.*/clightning-\(v[0-9.rc\-]*\)$,\1,p')} +MAKEPAR=${MAKEPAR:-1} + +# eg. ## [0.6.3] - 2019-01-09: "The Smallblock Conspiracy" +# Skip 'v' here in $VERSION +MTIME=${FORCE_MTIME:-$(sed -n "s/^## \\[${VERSION#v}\\] - \\([-0-9]*\\).*/\\1/p" < CHANGELOG.md)} +if [ -z "$MTIME" ]; then + echo "No date found for $VERSION in CHANGELOG.md" >&2 + exit 1 +fi + +echo "Repro Version: $VERSION" +echo "Repro mTime: $MTIME" +echo "Repro Platform: $PLATFORM" + +if grep ^deb /etc/apt/sources.list | grep -- '-\(updates\|security\)'; then + echo Please disable security and updates in /etc/apt/sources.list >&2 + exit 1 +fi + +DOWNLOAD='sudo apt -y --no-install-recommends --reinstall -d install' +PKGS='autoconf automake libtool make gcc libsqlite3-dev zlib1g-dev libsodium-dev' +INST='sudo dpkg -i' + +case "$PLATFORM" in + Ubuntu-22.04) + cat > /tmp/SHASUMS < /tmp/SHASUMS < /tmp/SHASUMS <&2 + exit 1 + ;; +esac + +# Download the packages +# shellcheck disable=SC2086 +$DOWNLOAD $PKGS + +# Make sure versions match, and exactly. +sha256sum -c /tmp/SHASUMS + +# Install them +# shellcheck disable=SC2046 +$INST $(cut -c66- < /tmp/SHASUMS) + +# Build ready for packaging. +# Once everyone has gcc8, we can use CC="gcc -ffile-prefix-map=$(pwd)=/home/clightning" +./configure --prefix=/usr CC="gcc -fdebug-prefix-map=$(pwd)=/home/clightning" +# libwally wants "python". Seems to work to force it here. +make -j"$MAKEPAR" PYTHON_VERSION=3 VERSION="$VERSION" +# VERSION must be passed here too: the Makefile falls back to git describe, +# which inside the builder sees the checked-out commit, not the release tag. +# Without it, --force-version names the tarball but the binaries are stamped +# with the branch version. +make -j"$MAKEPAR" install DESTDIR=inst/ VERSION="$VERSION" + +cd inst && tar --sort=name \ + --mtime="$MTIME 00:00Z" \ + --owner=0 --group=0 --numeric-owner -cvaf ../clightning-"$VERSION-$PLATFORM-$ARCH".tar.xz . diff --git a/tools/repro-build.armv7.sh b/tools/repro-build.armv7.sh new file mode 100755 index 000000000000..5c3b789abfb1 --- /dev/null +++ b/tools/repro-build.armv7.sh @@ -0,0 +1,313 @@ +#! /bin/sh +# ARMv7 (32-bit, hard float) variant of repro-build.sh. +# Unlike the amd64 and arm64 scripts this one cross-compiles: it runs in the +# amd64 builder image and uses Ubuntu's arm-linux-gnueabihf toolchain plus +# armhf libraries from ports.ubuntu.com, all pinned below. A native armv7 +# build is not possible: the vendored protoc used by cln-grpc has no armv7 +# binary. +# +# The build runs a few armv7 programs (configure tests, tools/headerversions), +# so the host needs qemu-arm registered with binfmt_misc, e.g. +# docker run --privileged --rm tonistiigi/binfmt --install arm + +set -e + +LANG=C +LC_ALL=C +export LANG LC_ALL + +for arg; do + case "$arg" in + --force-version=*) + FORCE_VERSION=${arg#*=} + ;; + --force-mtime=*) + FORCE_MTIME=${arg#*=} + ;; + --help) + echo "Usage: [--force-version=] [--force-mtime=YYYY-MM-DD]" + exit 0 + ;; + *) + echo "Unknown arg $arg" >&2 + exit 1 + ;; + esac + shift +done + +# Taken from https://unix.stackexchange.com/questions/6345/how-can-i-get-distribution-name-and-version-number-in-a-simple-shell-script +if [ -f /etc/os-release ]; then + # freedesktop.org and systemd + # shellcheck disable=SC1091 + . /etc/os-release + OS=$NAME + VER=$VERSION_ID +elif command -v lsb_release >/dev/null 2>&1; then + # linuxbase.org + OS=$(lsb_release -si) + VER=$(lsb_release -sr) +elif [ -f /etc/lsb-release ]; then + # For some versions of Debian/Ubuntu without lsb_release command + # shellcheck disable=SC1091 + . /etc/lsb-release + OS=$DISTRIB_ID + VER=$DISTRIB_RELEASE +elif [ -f /etc/debian_version ]; then + # Older Debian/Ubuntu/etc. + OS=Debian + VER=$(cat /etc/debian_version) +else + # Fall back to uname, e.g. "Linux ", also works for BSD, etc. + OS=$(uname -s) + VER=$(uname -r) +fi + +ARCH=armv7 +PLATFORM="$OS"-"$VER" +VERSION=${FORCE_VERSION:-$(git describe --tags --always --dirty=-modded --abbrev=7 2>/dev/null || pwd | sed -n 's,.*/clightning-\(v[0-9.rc\-]*\)$,\1,p')} +MAKEPAR=${MAKEPAR:-1} + +# eg. ## [0.6.3] - 2019-01-09: "The Smallblock Conspiracy" +# Skip 'v' here in $VERSION +MTIME=${FORCE_MTIME:-$(sed -n "s/^## \\[${VERSION#v}\\] - \\([-0-9]*\\).*/\\1/p" < CHANGELOG.md)} +if [ -z "$MTIME" ]; then + echo "No date found for $VERSION in CHANGELOG.md" >&2 + exit 1 +fi + +echo "Repro Version: $VERSION" +echo "Repro mTime: $MTIME" +echo "Repro Platform: $PLATFORM" + +if grep ^deb /etc/apt/sources.list | grep -- '-\(updates\|security\)'; then + echo Please disable security and updates in /etc/apt/sources.list >&2 + exit 1 +fi + +# Same release pocket as the main archive (updates and security are disabled +# above), but armhf lives on the ports mirror. +# VERSION_CODENAME comes from /etc/os-release, sourced above. +CODENAME=${VERSION_CODENAME:?no VERSION_CODENAME in /etc/os-release} +sudo dpkg --add-architecture armhf +sudo sed -i 's/^deb \(http\)/deb [arch=amd64] \1/' /etc/apt/sources.list +echo "deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports $CODENAME main universe" | sudo tee -a /etc/apt/sources.list +sudo apt-get update + +DOWNLOAD='sudo apt -y --no-install-recommends --reinstall -d install' +PKGS='autoconf automake libtool make gcc-arm-linux-gnueabihf libc6-dev:armhf libsqlite3-dev:armhf zlib1g-dev:armhf libsodium-dev:armhf libpq-dev:armhf' +INST='sudo dpkg -i' + +case "$PLATFORM" in + Ubuntu-22.04) + cat > /tmp/SHASUMS < /tmp/SHASUMS < /tmp/SHASUMS <&2 + exit 1 + ;; +esac + +# Download the packages +# shellcheck disable=SC2086 +$DOWNLOAD $PKGS + +# Make sure versions match, and exactly. +sha256sum -c /tmp/SHASUMS + +# Install them +# shellcheck disable=SC2046 +$INST $(cut -c66- < /tmp/SHASUMS) + +RUST_TARGET=armv7-unknown-linux-gnueabihf +rustup target add "$RUST_TARGET" +CARGO_BUILD_TARGET=$RUST_TARGET +CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc +export CARGO_BUILD_TARGET CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER + +# MAKE_HOST and BUILD give the bundled autotools libraries the right --host; +# TARGET tells the Makefile where cargo puts the armv7 binaries. +CROSS="MAKE_HOST=arm-linux-gnueabihf BUILD=x86_64-linux-gnu TARGET=$RUST_TARGET" + +# Build ready for packaging. +./configure --prefix=/usr CC="arm-linux-gnueabihf-gcc -fdebug-prefix-map=$(pwd)=/home/clightning" +# shellcheck disable=SC2086 +make -j"$MAKEPAR" PYTHON_VERSION=3 VERSION="$VERSION" $CROSS +# shellcheck disable=SC2086 +make -j"$MAKEPAR" install DESTDIR=inst/ VERSION="$VERSION" $CROSS + +cd inst && tar --sort=name \ + --mtime="$MTIME 00:00Z" \ + --owner=0 --group=0 --numeric-owner -cvaf ../clightning-"$VERSION-$PLATFORM-$ARCH".tar.xz . diff --git a/tools/sign-release-arm.sh b/tools/sign-release-arm.sh new file mode 100755 index 000000000000..e1f46c0d8f2a --- /dev/null +++ b/tools/sign-release-arm.sh @@ -0,0 +1,71 @@ +#! /bin/sh +# Checksum and sign the release tarballs of one ARM architecture. +# +# build-release.sh's sign target cannot be used for these: it hardcodes +# `cd release/` rather than honouring RELEASEDIR, and globs the zip, so +# running it for an ARM build would rewrite the amd64 manifest instead. +# +# Each architecture gets its own manifest, SHA256SUMS--: +# arm64 builds natively and armv7 is cross-compiled, so they are reproduced +# (and co-signed) separately. +# +# Usage: tools/sign-release-arm.sh v26.06.9 arm64|armv7 [gpg-key-id] +set -e + +VERSION=${1:-} +ARCH=${2:-} +KEY=${3:-} +case "$ARCH" in + arm64|armv7) ;; + *) + echo "Usage: $0 arm64|armv7 [gpg-key-id]" >&2 + exit 1 + ;; +esac +if [ -z "$VERSION" ]; then + echo "Usage: $0 arm64|armv7 [gpg-key-id]" >&2 + exit 1 +fi + +RELEASEDIR="$(pwd)/release-$ARCH" +SUMS="SHA256SUMS-$VERSION-$ARCH" + +[ -d "$RELEASEDIR" ] || { echo "No $RELEASEDIR: build the $ARCH tarballs first" >&2; exit 1; } +cd "$RELEASEDIR" + +set -- clightning-"$VERSION"-*-"$ARCH".tar.* +[ -e "$1" ] || { echo "No $ARCH tarballs for $VERSION in $RELEASEDIR" >&2; exit 1; } + +# sha256sum is GNU; macOS ships shasum. +if command -v sha256sum >/dev/null; then + SHA256SUM="sha256sum" +else + SHA256SUM="shasum -a 256" +fi + +echo "Checksumming $# $ARCH tarball(s) into $SUMS" +$SHA256SUM "$@" > "$SUMS" +cat "$SUMS" + +if [ -z "$KEY" ]; then + KEY=$(gpgconf --list-options gpg | awk -F: '$1 == "default-key" {print $10}' | tr -d '"') +fi +[ -n "$KEY" ] || { echo "No signing key: pass one, or set default-key in gpg.conf" >&2; exit 1; } + +echo "Signing $SUMS with $KEY" +gpg -sb --armor --default-key "$KEY" -o "$SUMS.asc" "$SUMS" +gpg --verify "$SUMS.asc" "$SUMS" +echo "Signed: $RELEASEDIR/$SUMS.asc" + +# Aggregating other people's signatures +# +# Concatenating .asc files can silently produce a file gpg only half-reads: +# an armor block whose BEGIN line is not followed by a blank line yields +# "invalid armor header" and a CRC error, and gpg then verifies only the +# blocks it managed to parse. Normalise before appending: +# +# awk '/^-----BEGIN PGP SIGNATURE-----$/{print; getline l; if (l != "") print ""; print l; next} {print}' \ +# theirs.asc >> SHA256SUMS--.asc +# +# Then confirm the count is what you expect: +# gpg --verify SHA256SUMS--.asc SHA256SUMS-- 2>&1 | grep -c "Good signature"