From eb10ac5e115b628a976a223bc8644eb3bd9b62f2 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 16 Jul 2026 14:22:00 -0400 Subject: [PATCH 1/9] ci/nix-install-ephemeral: Clean up action's nix-extra.conf * No need for sudo to write to /tmp * Put the github token into a separate file to avoid leaking by mistake * Do not set extra-system-features = kvm until we know we have a kvm device * Drop sourcing the shell env since its not actually used in this step --- .../actions/nix-install-ephemeral/action.yml | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index 638111853a..83bd67100e 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -44,34 +44,35 @@ runs: - name: Install Nix shell: bash run: | - sudo tee /tmp/nix-extra.conf > /dev/null <<'NIXCONF' + sudo tee /etc/nix/github.nix.conf >/dev/null <<'EOF' + access-tokens = github.com=${{ github.token }} + EOF + chmod 400 /etc/nix/github.nix.conf + cat >/tmp/nix-extra.conf <<'NIXCONF' always-allow-substitutes = true - extra-experimental-features = nix-command flakes - extra-system-features = kvm + extra-experimental-features = flakes nix-command + extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com + extra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= max-jobs = 4 - substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com - trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= NIXCONF - if [ "${{ inputs.push-to-cache }}" = "true" ]; then - echo "post-build-hook = /etc/nix/upload-to-cache.sh" | sudo tee -a /tmp/nix-extra.conf > /dev/null + + if [[ -e /dev/kvm ]]; then + echo 'extra-system-features = kvm' >>/tmp/nix-extra.conf + fi + + if [[ "${{ inputs.push-to-cache }}" == true ]]; then + echo 'post-build-hook = /etc/nix/upload-to-cache.sh' >>/tmp/nix-extra.conf fi + echo '!include /etc/nix/github.nix.conf' >>/tmp/nix-extra.conf + curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- --daemon --yes --nix-extra-conf-file /tmp/nix-extra.conf cat /etc/nix/nix.conf # Add nix to PATH for subsequent steps echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH" - # Source the daemon profile so nix works in this step too - . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh - - name: Configure Nix with GitHub token - shell: bash - env: - GH_TOKEN: ${{ github.token }} - run: | - echo "access-tokens = github.com=$GH_TOKEN" | sudo tee -a /etc/nix/nix.conf > /dev/null - sudo systemctl restart nix-daemon || true - name: Print Nix version shell: bash run: nix --version From 1a4a13588beab5120466b0ac930250bb48a586fb Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 16 Jul 2026 14:22:00 -0400 Subject: [PATCH 2/9] ci/nix-install-ephemeral: Add multi-user input and single-user install support This way we can use this action in the qemu image build workflow that runs in dind and requires single-user install. --- .../actions/nix-install-ephemeral/action.yml | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index 83bd67100e..a0557554dc 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -1,14 +1,18 @@ name: 'Install Nix on ephemeral runners' description: 'Installs Nix and sets up AWS credentials to push to the Nix binary cache' inputs: + aws-region: + description: 'AWS region for the Nix binary cache S3 bucket' + required: false + default: 'us-east-1' push-to-cache: description: 'Whether to push build outputs to the Nix binary cache' required: false default: 'false' - aws-region: - description: 'AWS region for the Nix binary cache S3 bucket' + multi-user: + description: Whether to install nix in multi-user mode (default true) or single-user mode required: false - default: 'us-east-1' + default: 'true' runs: using: 'composite' steps: @@ -44,11 +48,26 @@ runs: - name: Install Nix shell: bash run: | - sudo tee /etc/nix/github.nix.conf >/dev/null <<'EOF' - access-tokens = github.com=${{ github.token }} - EOF - chmod 400 /etc/nix/github.nix.conf - cat >/tmp/nix-extra.conf <<'NIXCONF' + if [[ "${{ inputs.multi-user }}" == true ]]; then + daemon=--daemon + nixconfdir=/etc/nix + path=/nix/var/nix/profiles/default/bin + tmpconf=/tmp/nix-extra.conf + + echo 'access-tokens = github.com=${{ github.token }}' | sudo tee $nixconfdir/github.nix.conf >/dev/null + sudo chmod 400 $nixconfdir/github.nix.conf + else + daemon=--no-daemon + nixconfdir=$HOME/.config/nix + path=$HOME/.nix-profile/bin + tmpconf=$nixconfdir/nix.conf # --nix-extra-conf-files is ignored for single-user installs + + mkdir -p "$(dirname $tmpconf)" + echo 'access-tokens = github.com=${{ github.token }}' | tee $nixconfdir/github.nix.conf >/dev/null + chmod 400 $nixconfdir/github.nix.conf + fi + + cat >$tmpconf <<'NIXCONF' always-allow-substitutes = true extra-experimental-features = flakes nix-command extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com @@ -58,21 +77,21 @@ runs: if [[ -e /dev/kvm ]]; then - echo 'extra-system-features = kvm' >>/tmp/nix-extra.conf + echo 'extra-system-features = kvm' >>$tmpconf fi if [[ "${{ inputs.push-to-cache }}" == true ]]; then - echo 'post-build-hook = /etc/nix/upload-to-cache.sh' >>/tmp/nix-extra.conf + echo 'post-build-hook = /etc/nix/upload-to-cache.sh' >>$tmpconf fi - echo '!include /etc/nix/github.nix.conf' >>/tmp/nix-extra.conf + echo "!include $nixconfdir/github.nix.conf" >>$tmpconf - curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- --daemon --yes --nix-extra-conf-file /tmp/nix-extra.conf + curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- $daemon --yes --nix-extra-conf-file $tmpconf - cat /etc/nix/nix.conf + sudo cat $nixconfdir/nix.conf # Add nix to PATH for subsequent steps - echo "/nix/var/nix/profiles/default/bin" >> "$GITHUB_PATH" + echo "$path" >> "$GITHUB_PATH" - name: Print Nix version shell: bash run: nix --version From 3100e11376992b2b49d32fbf357ab6424c15a1e7 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 16 Jul 2026 14:22:00 -0400 Subject: [PATCH 3/9] ci/nix-install-ephemeral: Add force-clean input Signals to force clean known nix paths. This is temporarily useful for native arm runners that have a pre-existing detsys nix install. --- .github/actions/nix-install-ephemeral/action.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/actions/nix-install-ephemeral/action.yml b/.github/actions/nix-install-ephemeral/action.yml index a0557554dc..af4c2292d4 100644 --- a/.github/actions/nix-install-ephemeral/action.yml +++ b/.github/actions/nix-install-ephemeral/action.yml @@ -5,6 +5,10 @@ inputs: description: 'AWS region for the Nix binary cache S3 bucket' required: false default: 'us-east-1' + force-clean: + description: Whether to delete pre-existing nix paths + required: false + default: 'false' push-to-cache: description: 'Whether to push build outputs to the Nix binary cache' required: false @@ -24,6 +28,13 @@ runs: aws-region: ${{ inputs.aws-region }} output-credentials: true role-duration-seconds: 7200 + - name: Force Clean Pre-Existing Nix Paths + if: inputs.force-clean == 'true' + shell: bash + run: sudo rm -rf /nix /etc/nix "$HOME/.nix-profile" "$HOME/.nix-channels" "$HOME/.config/nix" + - name: Ensure /etc/nix exists + shell: bash + run: sudo mkdir -p /etc/nix - name: Setup AWS credentials for Nix if: ${{ inputs.push-to-cache == 'true' }} shell: bash @@ -32,7 +43,6 @@ runs: sudo -H aws configure set aws_secret_access_key $AWS_SECRET_ACCESS_KEY sudo -H aws configure set aws_session_token $AWS_SESSION_TOKEN sudo -H aws configure set region ${{ inputs.aws-region }} - sudo mkdir -p /etc/nix sudo -E python -c "import os; file = open('/etc/nix/nix-secret-key', 'w'); file.write(os.environ['NIX_SIGN_SECRET_KEY']); file.close()" cat << 'EOF' | sudo tee /etc/nix/upload-to-cache.sh > /dev/null #!/usr/bin/env bash From a65dc4b9d681320ec8a7905f276d0f5ca141868b Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Tue, 14 Jul 2026 22:05:53 -0400 Subject: [PATCH 4/9] packer/qemu: Wrap image build in nix --- .github/workflows/qemu-image-build.yml | 43 +++++---------- .gitignore | 4 +- Dockerfile-kubernetes | 2 +- .../build-qemu-image/build-qemu-image.sh | 36 +++++++++++++ nix/packages/build-qemu-image/default.nix | 23 ++++++++ nix/packages/default.nix | 1 + qemu-arm64-nix.pkr.hcl | 53 +++++++------------ 7 files changed, 98 insertions(+), 64 deletions(-) create mode 100644 nix/packages/build-qemu-image/build-qemu-image.sh create mode 100644 nix/packages/build-qemu-image/default.nix diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index f3f9e417a8..d14d1bc1bd 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -3,11 +3,11 @@ name: Build QEMU image on: push: paths: - - '.github/workflows/qemu-image-build.yml' - - 'qemu-arm64-nix.pkr.hcl' - - 'common-nix.vars.pkr.hcl' - - 'ansible/vars.yml' - - 'scripts/*' + - .github/workflows/qemu-image-build.yml + - ansible/vars.yml + - nix/packages/build-qemu-image/* + - qemu-arm64-nix.pkr.hcl + - scripts/* workflow_dispatch: permissions: @@ -64,35 +64,20 @@ jobs: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV echo "EXECUTION_ID=${{ github.run_id }}-${{ matrix.postgres_version }}" >> $GITHUB_ENV - - name: Generate common-nix.vars.pkr.hcl - run: | - curl -L https://github.com/mikefarah/yq/releases/download/v4.45.1/yq_linux_arm64 -o yq && chmod +x yq - PG_VERSION=$(./yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) - PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes - echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl - echo 'postgres_major_version = "'$POSTGRES_MAJOR_VERSION'"' >> common-nix.vars.pkr.hcl - # Ensure there's a newline at the end of the file - echo "" >> common-nix.vars.pkr.hcl - - # TODO (darora): not quite sure why I'm having to uninstall and re-install these deps, but the build fails w/o this - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get remove -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils - sudo apt-get install -y qemu-efi-aarch64 cloud-image-utils qemu-system-arm qemu-utils + - name: Install Nix + uses: ./.github/actions/nix-install-ephemeral + with: + force-clean: "true" + multi-user: "false" - name: Build QEMU artifact - run: | - make init - GIT_SHA=${{github.sha}} - export PACKER_LOG=1 - packer build -var "git_sha=${GIT_SHA}" -var-file="common-nix.vars.pkr.hcl" qemu-arm64-nix.pkr.hcl + run: nix run .#build-qemu-image "${{ matrix.postgres_version }}" - name: Grab release version id: process_release_version run: | - VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - echo "version=$VERSION" >> $GITHUB_OUTPUT + PG_VERSION=$(nix run nixpkgs#yq -- -r '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + echo "version=$PG_VERSION" >> $GITHUB_OUTPUT - name: configure aws credentials - staging if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release/') }} @@ -110,7 +95,7 @@ jobs: env: IMAGE_TAG: ${{ steps.process_release_version.outputs.version }} run: | - docker build -f Dockerfile-kubernetes -t "postgres:$IMAGE_TAG" . + docker build -f Dockerfile-kubernetes -t "postgres:$IMAGE_TAG" packer-work-qemu-* - name: Push docker image to Amazon ECR if: ${{ github.event_name == 'workflow_dispatch' || github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release/') }} diff --git a/.gitignore b/.gitignore index f24b95b1b3..bf3108970a 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ # ansible related ansible/image-manifest*.json +# packer related +packer-work-*/ + # python related *$py.class *.py[cod] @@ -21,5 +24,4 @@ venv/ #nix related common-nix.vars.pkr.hcl db/schema.sql -nixos.qcow2 result* diff --git a/Dockerfile-kubernetes b/Dockerfile-kubernetes index f7af10d248..61eda4e1dd 100644 --- a/Dockerfile-kubernetes +++ b/Dockerfile-kubernetes @@ -1,6 +1,6 @@ FROM alpine:3.23 -ADD ./output-cloudimg/packer-cloudimg /disk/image.qcow2 +ADD output-cloudimg/packer-cloudimg /disk/image.qcow2 RUN apk add --no-cache qemu-system-aarch64 qemu-img openssh-client aavmf virtiofsd \ && truncate -s 64M /root/varstore.img \ diff --git a/nix/packages/build-qemu-image/build-qemu-image.sh b/nix/packages/build-qemu-image/build-qemu-image.sh new file mode 100644 index 0000000000..99bf5c6150 --- /dev/null +++ b/nix/packages/build-qemu-image/build-qemu-image.sh @@ -0,0 +1,36 @@ +# shellcheck shell=bash + +# nix-built qemu resolves its libraries via RPATH; an inherited +# LD_LIBRARY_PATH (e.g. from a devshell) can inject a mismatched glibc and +# break `qemu-system-* --version`, so drop it. +unset LD_LIBRARY_PATH + +postgres_major_version=${1:-} +if [ -z "$postgres_major_version" ]; then + echo "Usage: build-qemu-image [arch]" >&2 + exit 1 +fi + +qemu=$(which qemu-system-aarch64) +code=${qemu%/bin/*}/share/qemu/edk2-aarch64-code.fd +vars=${qemu%/bin/*}/share/qemu/edk2-arm-vars.fd + +workdir=$(mktemp -d packer-work-qemu-XXXXXX) +install --mode 444 "$code" "$workdir/ovmf_code.fd" +install --mode 644 "$vars" "$workdir/ovmf_vars.fd" # qemu writes to the vars pflash during boot + +git_sha=${GIT_SHA:-$(git rev-parse HEAD)} +postgres_version=$(yq -r ".postgres_release[\"postgres$postgres_major_version\"]" ansible/vars.yml) + +# Build the cloud-init seed ISO before invoking packer. +cloud-localds "$workdir/seeds-cloudimg.iso" user-data-cloudimg meta-data + +export LC_ALL=C.UTF-8 +export TZ=UTC +packer init qemu-arm64-nix.pkr.hcl +PACKER_LOG=${PACKER_LOG:-1} packer build \ + -var "git_sha=$git_sha" \ + -var "postgres-version=$postgres_version" \ + -var "postgres_major_version=$postgres_major_version" \ + -var "workdir=$workdir" \ + qemu-arm64-nix.pkr.hcl diff --git a/nix/packages/build-qemu-image/default.nix b/nix/packages/build-qemu-image/default.nix new file mode 100644 index 0000000000..94762f6f47 --- /dev/null +++ b/nix/packages/build-qemu-image/default.nix @@ -0,0 +1,23 @@ +{ + cloud-utils, + coreutils, + gitMinimal, + packer, + qemu, + writeShellApplication, + yq, +}: +writeShellApplication { + name = "build-qemu-image"; + + runtimeInputs = [ + cloud-utils + coreutils + gitMinimal + packer + qemu + yq + ]; + + text = builtins.readFile ./build-qemu-image.sh; +} diff --git a/nix/packages/default.nix b/nix/packages/default.nix index a452a9b784..c3b1ae012e 100644 --- a/nix/packages/default.nix +++ b/nix/packages/default.nix @@ -44,6 +44,7 @@ packages = ( { build-ami = pkgs.callPackage ./build-ami.nix { packer = self'.packages.packer; }; + build-qemu-image = pkgs.callPackage ./build-qemu-image { packer = self'.packages.packer; }; build-test-ami = pkgs.callPackage ./build-test-ami.nix { packer = self'.packages.packer; }; cleanup-ami = pkgs.callPackage ./cleanup-ami.nix { }; dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; }; diff --git a/qemu-arm64-nix.pkr.hcl b/qemu-arm64-nix.pkr.hcl index 61c0bdc4e5..c366741210 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu-arm64-nix.pkr.hcl @@ -12,6 +12,11 @@ variable "postgres_major_version" { default = "" } +# Working dir to pick up non-source files and put output files +variable "workdir" { + type = string +} + packer { required_plugins { qemu = { @@ -21,36 +26,18 @@ packer { } } -source "null" "dependencies" { - communicator = "none" -} - -build { - name = "cloudimg.deps" - sources = [ - "source.null.dependencies" - ] - - provisioner "shell-local" { - inline = [ - "cp /usr/share/AAVMF/AAVMF_VARS.fd AAVMF_VARS.fd", - "cloud-localds seeds-cloudimg.iso user-data-cloudimg meta-data" - ] - inline_shebang = "/bin/bash -e" - } -} - source "qemu" "cloudimg" { - boot_wait = "2s" - cpus = 8 - disk_image = true - disk_size = "15G" - format = "qcow2" - headless = true - http_directory = "http" - iso_checksum = "file:https://cloud-images.ubuntu.com/minimal/releases/noble/release/SHA256SUMS" - iso_url = "https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-arm64.img" - memory = 40000 + boot_wait = "2s" + cpus = 8 + disk_image = true + disk_size = "15G" + format = "qcow2" + headless = true + http_directory = "http" + iso_checksum = "file:https://cloud-images.ubuntu.com/minimal/releases/noble/release/SHA256SUMS" + iso_url = "https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-arm64.img" + memory = 40000 + output_directory = "${var.workdir}/output-cloudimg" qemu_img_args { convert = ["-o", "compression_type=zstd"] } @@ -59,10 +46,10 @@ source "qemu" "cloudimg" { ["-machine", "virt,gic-version=3"], ["-cpu", "host"], ["-device", "virtio-gpu-pci"], - ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=/usr/share/AAVMF/AAVMF_CODE.fd"], - ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=AAVMF_VARS.fd"], - ["-drive", "file=output-cloudimg/packer-cloudimg,if=virtio,format=qcow2,discard=on,detect-zeroes=unmap"], - ["-drive", "file=seeds-cloudimg.iso,format=raw"], + ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=${var.workdir}/ovmf_code.fd"], + ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=${var.workdir}/ovmf_vars.fd"], + ["-drive", "file=${var.workdir}/output-cloudimg/packer-cloudimg,if=virtio,format=qcow2,discard=on,detect-zeroes=unmap"], + ["-drive", "file=${var.workdir}/seeds-cloudimg.iso,format=raw"], ["--enable-kvm"] ] shutdown_command = "sudo -S shutdown -P now" From 2d00e65f2f4859e9830cfbb860c5bfcc3cd3e6ac Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Wed, 15 Jul 2026 09:07:36 -0400 Subject: [PATCH 5/9] packer/qemu: Support x86_64 Same as before where host must match target. This is nice to test x86_64 specific playbook tasks. Need to add virtio for the seeds iso because on x86_64 the virtual hardware driver used isn't loaded by default in the ubuntu image and so packer can't connect via ssh. --- .github/workflows/qemu-image-build.yml | 4 +-- .../build-qemu-image/build-qemu-image.sh | 28 +++++++++++++++---- qemu-arm64-nix.pkr.hcl => qemu.pkr.hcl | 21 +++++++++++--- scripts/90-cleanup-qemu.sh | 5 +++- 4 files changed, 46 insertions(+), 12 deletions(-) rename qemu-arm64-nix.pkr.hcl => qemu.pkr.hcl (88%) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index d14d1bc1bd..e9c51fea99 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -6,7 +6,7 @@ on: - .github/workflows/qemu-image-build.yml - ansible/vars.yml - nix/packages/build-qemu-image/* - - qemu-arm64-nix.pkr.hcl + - qemu.pkr.hcl - scripts/* workflow_dispatch: @@ -71,7 +71,7 @@ jobs: multi-user: "false" - name: Build QEMU artifact - run: nix run .#build-qemu-image "${{ matrix.postgres_version }}" + run: nix run .#build-qemu-image "${{ matrix.postgres_version }}" arm64 - name: Grab release version id: process_release_version diff --git a/nix/packages/build-qemu-image/build-qemu-image.sh b/nix/packages/build-qemu-image/build-qemu-image.sh index 99bf5c6150..3c11358c21 100644 --- a/nix/packages/build-qemu-image/build-qemu-image.sh +++ b/nix/packages/build-qemu-image/build-qemu-image.sh @@ -11,9 +11,24 @@ if [ -z "$postgres_major_version" ]; then exit 1 fi -qemu=$(which qemu-system-aarch64) -code=${qemu%/bin/*}/share/qemu/edk2-aarch64-code.fd -vars=${qemu%/bin/*}/share/qemu/edk2-arm-vars.fd +declare -A host2arch=([aarch64]=arm64 [x86_64]=amd64) +host=${host2arch[$(uname -m)]} +arch=${2:-$host} +case $arch in +amd64) + qemu=$(which qemu-system-x86_64) + code=${qemu%/bin/*}/share/qemu/edk2-x86_64-code.fd + vars=${qemu%/bin/*}/share/qemu/edk2-i386-vars.fd + machine=q35 + ;; +arm64) + qemu=$(which qemu-system-aarch64) + code=${qemu%/bin/*}/share/qemu/edk2-aarch64-code.fd + vars=${qemu%/bin/*}/share/qemu/edk2-arm-vars.fd + machine=virt,gic-version=3 + ;; +*) echo "Error: Invalid arch '$arch'. Must be 'amd64' or 'arm64'" >&2 && exit 1 ;; +esac workdir=$(mktemp -d packer-work-qemu-XXXXXX) install --mode 444 "$code" "$workdir/ovmf_code.fd" @@ -27,10 +42,13 @@ cloud-localds "$workdir/seeds-cloudimg.iso" user-data-cloudimg meta-data export LC_ALL=C.UTF-8 export TZ=UTC -packer init qemu-arm64-nix.pkr.hcl +packer init qemu.pkr.hcl PACKER_LOG=${PACKER_LOG:-1} packer build \ + -var "arch=$arch" \ -var "git_sha=$git_sha" \ + -var "machine=$machine" \ -var "postgres-version=$postgres_version" \ -var "postgres_major_version=$postgres_major_version" \ + -var "qemu_binary=$qemu" \ -var "workdir=$workdir" \ - qemu-arm64-nix.pkr.hcl + qemu.pkr.hcl diff --git a/qemu-arm64-nix.pkr.hcl b/qemu.pkr.hcl similarity index 88% rename from qemu-arm64-nix.pkr.hcl rename to qemu.pkr.hcl index c366741210..8ef5239d2c 100644 --- a/qemu-arm64-nix.pkr.hcl +++ b/qemu.pkr.hcl @@ -17,6 +17,19 @@ variable "workdir" { type = string } +# Arch-specific values supplied by build-qemu-image +variable "arch" { + type = string +} + +variable "machine" { + type = string +} + +variable "qemu_binary" { + type = string +} + packer { required_plugins { qemu = { @@ -35,21 +48,21 @@ source "qemu" "cloudimg" { headless = true http_directory = "http" iso_checksum = "file:https://cloud-images.ubuntu.com/minimal/releases/noble/release/SHA256SUMS" - iso_url = "https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-arm64.img" + iso_url = "https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-${var.arch}.img" memory = 40000 output_directory = "${var.workdir}/output-cloudimg" qemu_img_args { convert = ["-o", "compression_type=zstd"] } - qemu_binary = "qemu-system-aarch64" + qemu_binary = var.qemu_binary qemuargs = [ - ["-machine", "virt,gic-version=3"], + ["-machine", var.machine], ["-cpu", "host"], ["-device", "virtio-gpu-pci"], ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=${var.workdir}/ovmf_code.fd"], ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=${var.workdir}/ovmf_vars.fd"], ["-drive", "file=${var.workdir}/output-cloudimg/packer-cloudimg,if=virtio,format=qcow2,discard=on,detect-zeroes=unmap"], - ["-drive", "file=${var.workdir}/seeds-cloudimg.iso,format=raw"], + ["-drive", "file=${var.workdir}/seeds-cloudimg.iso,format=raw,if=virtio"], ["--enable-kvm"] ] shutdown_command = "sudo -S shutdown -P now" diff --git a/scripts/90-cleanup-qemu.sh b/scripts/90-cleanup-qemu.sh index c6cdda4bb3..911842e95b 100644 --- a/scripts/90-cleanup-qemu.sh +++ b/scripts/90-cleanup-qemu.sh @@ -34,10 +34,13 @@ elif [ -n "$(command -v apt-get)" ]; then libicu-dev \ libcgal-dev \ libgcc-9-dev \ - libgcc-8-dev \ ansible \ snapd + if [[ $(uname -m) == aarch64 ]]; then + apt-get -y remove --purge libgcc-8-dev + fi + # add-apt-repository --yes --remove ppa:ansible/ansible source /etc/os-release From 67c2cd55cdd63f9a01e0985c54f41c780dad439e Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 16 Jul 2026 13:03:20 -0400 Subject: [PATCH 6/9] packer/qemu: Support building on Darwin --- nix/packages/build-qemu-image/build-qemu-image.sh | 10 ++++++++-- qemu.pkr.hcl | 4 +--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/nix/packages/build-qemu-image/build-qemu-image.sh b/nix/packages/build-qemu-image/build-qemu-image.sh index 3c11358c21..6f0da3f5cd 100644 --- a/nix/packages/build-qemu-image/build-qemu-image.sh +++ b/nix/packages/build-qemu-image/build-qemu-image.sh @@ -11,6 +11,12 @@ if [ -z "$postgres_major_version" ]; then exit 1 fi +case $(uname -o) in +Darwin) accel=hvf ;; +GNU/Linux) accel=kvm ;; +*) echo "Error: Invalid os '$(uname -o)'. Must be Darwin or Linux" >&2 && exit 1 ;; +esac + declare -A host2arch=([aarch64]=arm64 [x86_64]=amd64) host=${host2arch[$(uname -m)]} arch=${2:-$host} @@ -19,13 +25,13 @@ amd64) qemu=$(which qemu-system-x86_64) code=${qemu%/bin/*}/share/qemu/edk2-x86_64-code.fd vars=${qemu%/bin/*}/share/qemu/edk2-i386-vars.fd - machine=q35 + machine=q35,accel=$accel ;; arm64) qemu=$(which qemu-system-aarch64) code=${qemu%/bin/*}/share/qemu/edk2-aarch64-code.fd vars=${qemu%/bin/*}/share/qemu/edk2-arm-vars.fd - machine=virt,gic-version=3 + machine=virt,accel=$accel,gic-version=max,highmem=on ;; *) echo "Error: Invalid arch '$arch'. Must be 'amd64' or 'arm64'" >&2 && exit 1 ;; esac diff --git a/qemu.pkr.hcl b/qemu.pkr.hcl index 8ef5239d2c..b3b8e974c9 100644 --- a/qemu.pkr.hcl +++ b/qemu.pkr.hcl @@ -49,7 +49,7 @@ source "qemu" "cloudimg" { http_directory = "http" iso_checksum = "file:https://cloud-images.ubuntu.com/minimal/releases/noble/release/SHA256SUMS" iso_url = "https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-24.04-minimal-cloudimg-${var.arch}.img" - memory = 40000 + memory = 4096 # MiB output_directory = "${var.workdir}/output-cloudimg" qemu_img_args { convert = ["-o", "compression_type=zstd"] @@ -63,7 +63,6 @@ source "qemu" "cloudimg" { ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=${var.workdir}/ovmf_vars.fd"], ["-drive", "file=${var.workdir}/output-cloudimg/packer-cloudimg,if=virtio,format=qcow2,discard=on,detect-zeroes=unmap"], ["-drive", "file=${var.workdir}/seeds-cloudimg.iso,format=raw,if=virtio"], - ["--enable-kvm"] ] shutdown_command = "sudo -S shutdown -P now" ssh_handshake_attempts = 500 @@ -72,7 +71,6 @@ source "qemu" "cloudimg" { ssh_username = "ubuntu" ssh_wait_timeout = "1h" use_backing_file = false - accelerator = "kvm" } build { From 05ee0e472d454c878afa708c0059faccbb29c356 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Tue, 21 Jul 2026 17:11:26 -0400 Subject: [PATCH 7/9] packer/qemu: Add software based virtualization support This is _really_ slow (~10x) compared to native, *but* can be useful for devs anyway, no reason not to allow this IMO. I've added an escape hatch for CI to opt into so that we don't mistakingly try to build images one the wrong arch and have to wait forever for them. --- .github/workflows/qemu-image-build.yml | 2 +- nix/packages/build-qemu-image/build-qemu-image.sh | 14 ++++++++++++-- qemu.pkr.hcl | 6 +++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index e9c51fea99..4210ba65ab 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -71,7 +71,7 @@ jobs: multi-user: "false" - name: Build QEMU artifact - run: nix run .#build-qemu-image "${{ matrix.postgres_version }}" arm64 + run: BUILD_QEMU_IMAGE_HW_VIRT_ONLY=1 nix run .#build-qemu-image "${{ matrix.postgres_version }}" arm64 - name: Grab release version id: process_release_version diff --git a/nix/packages/build-qemu-image/build-qemu-image.sh b/nix/packages/build-qemu-image/build-qemu-image.sh index 6f0da3f5cd..31f1514f86 100644 --- a/nix/packages/build-qemu-image/build-qemu-image.sh +++ b/nix/packages/build-qemu-image/build-qemu-image.sh @@ -25,17 +25,26 @@ amd64) qemu=$(which qemu-system-x86_64) code=${qemu%/bin/*}/share/qemu/edk2-x86_64-code.fd vars=${qemu%/bin/*}/share/qemu/edk2-i386-vars.fd - machine=q35,accel=$accel ;; arm64) qemu=$(which qemu-system-aarch64) code=${qemu%/bin/*}/share/qemu/edk2-aarch64-code.fd vars=${qemu%/bin/*}/share/qemu/edk2-arm-vars.fd - machine=virt,accel=$accel,gic-version=max,highmem=on ;; *) echo "Error: Invalid arch '$arch'. Must be 'amd64' or 'arm64'" >&2 && exit 1 ;; esac +case $arch:$(uname -m) in +amd64:aarch64) machine=q35 cpu=qemu64 ;; +amd64:x86_64) machine=q35 cpu=host ;; +arm64:aarch64) machine=virt,gic-version=max,highmem=on cpu=host ;; +arm64:x86_64) machine=virt,gic-version=max,highmem=on cpu=cortex-a76 ;; +esac +machine+=,accel=$accel +if [[ -z ${BUILD_QEMU_IMAGE_HW_VIRT_ONLY:-} ]]; then + machine+=:tcg +fi + workdir=$(mktemp -d packer-work-qemu-XXXXXX) install --mode 444 "$code" "$workdir/ovmf_code.fd" install --mode 644 "$vars" "$workdir/ovmf_vars.fd" # qemu writes to the vars pflash during boot @@ -51,6 +60,7 @@ export TZ=UTC packer init qemu.pkr.hcl PACKER_LOG=${PACKER_LOG:-1} packer build \ -var "arch=$arch" \ + -var "cpu=$cpu" \ -var "git_sha=$git_sha" \ -var "machine=$machine" \ -var "postgres-version=$postgres_version" \ diff --git a/qemu.pkr.hcl b/qemu.pkr.hcl index b3b8e974c9..062c380c6a 100644 --- a/qemu.pkr.hcl +++ b/qemu.pkr.hcl @@ -22,6 +22,10 @@ variable "arch" { type = string } +variable "cpu" { + type = string +} + variable "machine" { type = string } @@ -57,7 +61,7 @@ source "qemu" "cloudimg" { qemu_binary = var.qemu_binary qemuargs = [ ["-machine", var.machine], - ["-cpu", "host"], + ["-cpu", var.cpu], ["-device", "virtio-gpu-pci"], ["-drive", "if=pflash,format=raw,id=ovmf_code,readonly=on,file=${var.workdir}/ovmf_code.fd"], ["-drive", "if=pflash,format=raw,id=ovmf_vars,file=${var.workdir}/ovmf_vars.fd"], From 32726a087f7a48454d7f19feabc81fd7e4e64667 Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Sat, 18 Jul 2026 16:01:04 -0400 Subject: [PATCH 8/9] packer/qemu: Upgrade 1.1.5 -> 1.1.6 Fixes packer-qemu-plugin 100% cpu usage across all cores (similar to seen with the amazon ebssurrogate plugin). CPU usage seen with gnutime -v, before: ``` User time (seconds): 4703.36 System time (seconds): 259.53 Percent of CPU this job got: 781% Elapsed (wall clock) time (h:mm:ss or m:ss): 10:35.18 ``` after: ``` User time (seconds): 732.49 System time (seconds): 310.12 Percent of CPU this job got: 209% Elapsed (wall clock) time (h:mm:ss or m:ss): 8:17.33 ``` --- qemu.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qemu.pkr.hcl b/qemu.pkr.hcl index 062c380c6a..02a6ff38fc 100644 --- a/qemu.pkr.hcl +++ b/qemu.pkr.hcl @@ -38,7 +38,7 @@ packer { required_plugins { qemu = { source = "github.com/hashicorp/qemu" - version = "1.1.5" + version = "1.1.6" } } } From 2cd745f4570f9ae76208616cd07c59ba50864b0d Mon Sep 17 00:00:00 2001 From: Manuel Mendez Date: Thu, 16 Jul 2026 14:22:00 -0400 Subject: [PATCH 9/9] packer/qemu: Install qemu specific packages in qemu-bootstrap-nix.sh We already have a pre-ansible step to install packages that are missing compared to the ebs setup so let move these 2 there and avoid some weird qemu only package install tasks (not to mention that I'm not sure installing gpg in supabase-admin-agent file made much sense anyway). --- ansible/tasks/internal/supabase-admin-agent.yml | 7 ------- ansible/tasks/setup-system.yml | 8 -------- ebssurrogate/scripts/qemu-bootstrap-nix.sh | 15 ++++++++++++++- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/ansible/tasks/internal/supabase-admin-agent.yml b/ansible/tasks/internal/supabase-admin-agent.yml index 69e7861291..c5ab847362 100644 --- a/ansible/tasks/internal/supabase-admin-agent.yml +++ b/ansible/tasks/internal/supabase-admin-agent.yml @@ -68,13 +68,6 @@ arch: "arm64" when: platform == "arm64" -- name: install gpg explicitly for qemu artifacts - become: yes - apt: - pkg: - - gpg - when: qemu_mode is defined - - name: Download supabase-admin-agent archive get_url: url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-agent/v{{ supabase_admin_agent_release }}/supabase-admin-agent-{{ supabase_admin_agent_release }}-linux-{{ arch }}.tar.gz" diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index bb5cb9410a..8059ecd9c7 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -76,14 +76,6 @@ owner: 'root' group: 'root' -- name: Install other useful tools - ansible.builtin.apt: - pkg: - - less - update_cache: true - when: - - qemu_mode is defined - - name: Set the platform arch as a fact ansible.builtin.set_fact: platform: "{{ 'amd64' if ansible_facts['architecture'] == 'x86_64' else 'arm64' }}" diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 8a52260ee8..944d34d978 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -22,7 +22,20 @@ function waitfor_boot_finished { } function install_packages { - apt-get update && sudo apt-get install software-properties-common e2fsprogs nfs-common locales iptables arptables ebtables ufw logrotate -y + apt-get update + apt-get install -y \ + arptables \ + e2fsprogs \ + ebtables \ + gpg \ + iptables \ + less \ + locales \ + logrotate \ + nfs-common \ + software-properties-common \ + ufw \ + ; # TODO (darora): temporarily disabling while Launchpad is under ddos attack and very frequently timing out # add-apt-repository --yes --update ppa:ansible/ansible && sudo apt-get install ansible -y