diff --git a/ansible/files/adminapi.service.j2 b/ansible/files/adminapi.service.j2 index 305d1ac629..4e956703a5 100644 --- a/ansible/files/adminapi.service.j2 +++ b/ansible/files/adminapi.service.j2 @@ -14,7 +14,7 @@ Restart=always RestartSec=3 TimeoutStopSec=10 Environment="AWS_USE_DUALSTACK_ENDPOINT=true" -{% if qemu_mode is defined and qemu_mode %} +{% if qemu %} Environment="AWS_SDK_LOAD_CONFIG=true" {% endif %} StandardOutput=journal diff --git a/ansible/files/gotrue.service.j2 b/ansible/files/gotrue.service.j2 index 481ce7c986..4105e92c66 100644 --- a/ansible/files/gotrue.service.j2 +++ b/ansible/files/gotrue.service.j2 @@ -90,7 +90,7 @@ Environment=GOTRUE_RELOADING_POLLER_ENABLED=false # ensures only 1 reload operation occurs during a burst of config updates. Environment=GOTRUE_RELOADING_GRACE_PERIOD_INTERVAL=2s -{% if qemu_mode is defined and qemu_mode %} +{% if qemu %} # v3 does not use filesystem notifications for config reloads. Environment=GOTRUE_RELOADING_NOTIFY_ENABLED=false {% else %} diff --git a/ansible/files/postgres_exporter.service.j2 b/ansible/files/postgres_exporter.service.j2 index dcb107cb77..4cf067a9a2 100644 --- a/ansible/files/postgres_exporter.service.j2 +++ b/ansible/files/postgres_exporter.service.j2 @@ -6,7 +6,7 @@ StartLimitBurst=0 [Service] Type=simple -ExecStart=/opt/postgres_exporter/postgres_exporter --disable-settings-metrics --extend.query-path="/opt/postgres_exporter/queries.yml" --disable-default-metrics --no-collector.locks --no-collector.replication --no-collector.replication_slot --no-collector.stat_bgwriter --no-collector.stat_database --no-collector.stat_user_tables --no-collector.statio_user_tables --no-collector.wal {% if qemu_mode is defined and qemu_mode %}--no-collector.database {% endif %} +ExecStart=/opt/postgres_exporter/postgres_exporter --disable-settings-metrics --extend.query-path="/opt/postgres_exporter/queries.yml" --disable-default-metrics --no-collector.locks --no-collector.replication --no-collector.replication_slot --no-collector.stat_bgwriter --no-collector.stat_database --no-collector.stat_user_tables --no-collector.statio_user_tables --no-collector.wal {% if qemu %}--no-collector.database {% endif %} User=postgres Group=postgres diff --git a/ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.service.j2 b/ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.service.j2 index d93d87dfa4..d5647c8865 100644 --- a/ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.service.j2 +++ b/ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.service.j2 @@ -6,7 +6,7 @@ Requires=local-fs.target [Service] Type=oneshot -{% if qemu_mode is defined and qemu_mode %} +{% if qemu %} ExecStart=/opt/supabase-admin-agent/supabase-admin-agent --config /opt/supabase-admin-agent/config.yaml salt --apply --store-result --targets-key targetsv3-main.yaml User=root Group=root diff --git a/ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.timer.j2 b/ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.timer.j2 index 390e6da570..52a4a81558 100644 --- a/ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.timer.j2 +++ b/ansible/files/supabase_admin_agent_config/supabase-admin-agent_salt.timer.j2 @@ -4,7 +4,7 @@ Requires=supabase-admin-agent_salt.service [Timer] {# We're using a significantly lower frequency for triggering this agent on qemu images for the moment. Once we've performed additional validations re: the aggregate impact of running it more frequently, the frequency can be increased. #} -{% if qemu_mode is defined and qemu_mode %} +{% if qemu %} OnCalendar=*-*-* 0,6,12,18:00:00 RandomizedDelaySec={{ supabase_admin_agent_splay }} AccuracySec=1h diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 488dcc26fe..1a2543244a 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -2,6 +2,10 @@ become: yes pre_tasks: + - name: Set the platform arch as a fact + ansible.builtin.set_fact: + platform: "{{ 'amd64' if ansible_facts['architecture'] == 'x86_64' else 'arm64' }}" + - import_tasks: tasks/setup-system.yml vars_files: - ./vars.yml @@ -40,124 +44,119 @@ tags: - install-pgbouncer - install-supabase-internal - when: nixpkg_mode or stage2_nix - name: Install WAL-G import_tasks: tasks/setup-wal-g.yml - when: nixpkg_mode or stage2_nix - name: Install pgBackRest - import_tasks: + import_tasks: file: tasks/setup-pgbackrest.yml - when: - - nixpkg_mode or stage2_nix - name: Install Gotrue + when: not stage2 import_tasks: tasks/setup-gotrue.yml tags: - install-gotrue - install-supabase-internal - when: nixpkg_mode - + - name: Install PostgREST + when: not stage2 import_tasks: tasks/setup-postgrest.yml tags: - install-postgrest - install-supabase-internal - when: nixpkg_mode - name: Install Envoy + when: not stage2 import_tasks: tasks/setup-envoy.yml tags: - install-supabase-internal - when: nixpkg_mode - name: Install Kong + when: not stage2 import_tasks: tasks/setup-kong.yml tags: - install-supabase-internal - when: nixpkg_mode - name: Install nginx + when: not stage2 import_tasks: tasks/setup-nginx.yml tags: - install-supabase-internal - when: nixpkg_mode - name: Install Supabase specific content + when: not stage2 import_tasks: tasks/setup-supabase-internal.yml tags: - install-supabase-internal - when: nixpkg_mode - name: Fix IPv6 NDisc issues (disabled) + when: not stage2 and not qemu import_tasks: tasks/fix-ipv6-ndisc.yml tags: - install-supabase-internal - when: nixpkg_mode and (qemu_mode is undefined) - name: Adjust APT update intervals + when: not stage2 copy: src: files/apt_periodic dest: /etc/apt/apt.conf.d/10periodic - when: nixpkg_mode - + - name: Transfer init SQL files + when: stage2 copy: src: files/{{ item.source }} dest: /tmp/{{ item.dest }} loop: "{{ sql_files }}" - when: stage2_nix - name: Create postgres role + when: stage2 become: yes become_user: postgres shell: cmd: /usr/lib/postgresql/bin/psql --username=supabase_admin -d postgres -c "create role postgres superuser login; alter database postgres owner to postgres;" - when: stage2_nix - name: Execute init SQL files + when: stage2 become: yes become_user: postgres shell: cmd: /usr/lib/postgresql/bin/psql -f /tmp/{{ item.dest }} loop: "{{ sql_files }}" - when: stage2_nix - name: Delete SQL scripts + when: stage2 file: path: /tmp/{{ item.dest }} state: absent loop: "{{ sql_files }}" - when: stage2_nix - name: First boot optimizations + when: stage2 import_tasks: tasks/internal/optimizations.yml tags: - install-supabase-internal - when: stage2_nix - + - name: Finalize AMI + when: not stage2 import_tasks: tasks/finalize-ami.yml tags: - install-supabase-internal - when: nixpkg_mode - + - name: Enhance fail2ban + when: not stage2 import_tasks: tasks/setup-fail2ban.yml - when: nixpkg_mode - # Install EC2 instance connect # Only for AWS images - name: install EC2 instance connect + when: not qemu become: yes apt: pkg: - ec2-instance-connect tags: - aws-only - when: qemu_mode is undefined # Install this at the end to prevent it from kicking in during the apt process, causing conflicts - name: Install security tools @@ -172,53 +171,53 @@ import_tasks: tasks/clean-build-dependencies.yml - name: Run migrations + when: stage2 import_tasks: tasks/setup-migrations.yml tags: - migrations - when: stage2_nix - name: Run unit tests + when: stage2 import_tasks: tasks/test-image.yml tags: - unit-tests - when: stage2_nix - name: Purge snapd + when: stage2 become: yes shell: | apt autoremove -y --purge snapd - when: stage2_nix - name: Run permission checks + when: stage2 become: yes shell: | systemctl start postgresql.service - /usr/bin/python3 /tmp/ansible-playbook/ansible/files/permission_check.py {{ '--qemu' if qemu_mode is defined else '' }} + /usr/bin/python3 /tmp/ansible-playbook/ansible/files/permission_check.py {{ '--qemu' if qemu else '' }} systemctl stop postgresql.service - when: stage2_nix - name: Run fail2ban configuration checks + when: stage2 become: yes shell: | systemctl start fail2ban.service /usr/bin/python3 /tmp/ansible-playbook/ansible/files/fail2ban_check.py systemctl stop fail2ban.service - when: stage2_nix - name: Run supascan baseline validation + when: stage2 and not qemu and ansible_architecture != "x86_64" become: yes shell: | /bin/bash /tmp/ansible-playbook/ansible/files/supascan_ami.sh /tmp/ansible-playbook/audit-specs/baselines/ami-build - when: stage2_nix and qemu_mode is not defined and ansible_architecture != "x86_64" - name: Remove supascan after validation + when: stage2 become: yes shell: | sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile remove supascan" - when: stage2_nix - name: nix collect garbage + when: stage2 become: yes shell: | sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d" - when: stage2_nix diff --git a/ansible/tasks/finalize-ami.yml b/ansible/tasks/finalize-ami.yml index a1924a230f..902a180993 100644 --- a/ansible/tasks/finalize-ami.yml +++ b/ansible/tasks/finalize-ami.yml @@ -107,11 +107,10 @@ state: 'restarted' - name: import pgsodium_getkey script + when: stage2 ansible.builtin.template: dest: "{{ pg_bindir }}/pgsodium_getkey.sh" group: 'postgres' mode: '0700' owner: 'postgres' src: 'files/pgsodium_getkey_readonly.sh.j2' - when: - - stage2_nix diff --git a/ansible/tasks/internal/admin-api.yml b/ansible/tasks/internal/admin-api.yml index 0c8e4020e6..6affb840c7 100644 --- a/ansible/tasks/internal/admin-api.yml +++ b/ansible/tasks/internal/admin-api.yml @@ -27,19 +27,10 @@ shell: | chmod g+w /etc -- name: Setting arch (amd64) - set_fact: - arch: "amd64" - when: platform == "amd64" - -- name: Setting arch (arm) - set_fact: - arch: "arm64" - when: platform == "arm64" - - name: Download adminapi archive get_url: - url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-api/v{{ adminapi_release }}/supabase-admin-api_{{ adminapi_release }}_linux_{{ arch }}.tar.gz" + url: "{{ adminapi_artifacts[platform]['url'] }}" + checksum: "{{ adminapi_artifacts[platform]['checksum'] }}" dest: "/tmp/adminapi.tar.gz" timeout: 90 diff --git a/ansible/tasks/internal/admin-mgr.yml b/ansible/tasks/internal/admin-mgr.yml index 46a1231c84..2f1a73a8fa 100644 --- a/ansible/tasks/internal/admin-mgr.yml +++ b/ansible/tasks/internal/admin-mgr.yml @@ -1,16 +1,7 @@ -- name: Setting arch (x86) - set_fact: - arch: "amd64" - when: platform == "amd64" - -- name: Setting arch (arm) - set_fact: - arch: "arm64" - when: platform == "arm64" - - name: Download admin-mgr archive get_url: - url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/admin-mgr/v{{ adminmgr_release }}/admin-mgr_{{ adminmgr_release }}_linux_{{ arch }}.tar.gz" + url: "{{ adminmgr_artifacts[platform]['url'] }}" + checksum: "{{ adminmgr_artifacts[platform]['checksum'] }}" dest: "/tmp/admin-mgr.tar.gz" timeout: 90 register: download_result diff --git a/ansible/tasks/internal/install-aws-cli.yml b/ansible/tasks/internal/install-aws-cli.yml index 397c19ca15..d721c36a96 100644 --- a/ansible/tasks/internal/install-aws-cli.yml +++ b/ansible/tasks/internal/install-aws-cli.yml @@ -19,9 +19,10 @@ - name: AWS CLI - get ansible.builtin.get_url: + url: "{{ aws_cli_artifacts[platform]['url'] }}" + checksum: "{{ aws_cli_artifacts[platform]['checksum'] }}" dest: '/tmp/awscliv2.zip' timeout: 60 - url: "https://awscli.amazonaws.com/awscli-exe-linux-{{ 'aarch64' if platform == 'arm64' else 'x86_64' }}-{{ aws_cli_release }}.zip" register: download_result until: download_result is succeeded retries: 3 diff --git a/ansible/tasks/internal/install-salt.yml b/ansible/tasks/internal/install-salt.yml index 47436ded31..8f9ba32340 100644 --- a/ansible/tasks/internal/install-salt.yml +++ b/ansible/tasks/internal/install-salt.yml @@ -1,4 +1,4 @@ -- name: Add apt repository for Saltstack (arm) +- name: Add apt repository for Saltstack block: - name: Ensure /etc/apt/keyrings directory exists file: @@ -9,6 +9,7 @@ - name: salt gpg key ansible.builtin.get_url: url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public + checksum: sha256:36decef986477acb8ba2a1fc4041bcf9f22229ef6c939d0317c9e36a9d142b34 dest: /etc/apt/keyrings/salt-archive-keyring-2023.pgp mode: '0644' register: download_result @@ -18,35 +19,9 @@ - name: salt apt repo ansible.builtin.apt_repository: - repo: "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=arm64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" + repo: "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch={{ platform }}] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" filename: "salt" state: present - when: platform == "arm64" - -- name: Add apt repository for Saltstack (amd) - block: - - name: Ensure /etc/apt/keyrings directory exists - file: - path: /etc/apt/keyrings - state: directory - mode: "0755" - - - name: salt gpg key - ansible.builtin.get_url: - url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public - dest: /etc/apt/keyrings/salt-archive-keyring-2023.pgp - mode: '0644' - register: download_result - until: download_result is succeeded - retries: 3 - delay: 2 - - - name: salt apt repo - ansible.builtin.apt_repository: - repo: "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=amd64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" - filename: "salt" - state: present - when: platform == "amd64" - name: Salt minion install apt: diff --git a/ansible/tasks/internal/optimizations.yml b/ansible/tasks/internal/optimizations.yml index 0338fdd9a7..1fcdd83c1f 100644 --- a/ansible/tasks/internal/optimizations.yml +++ b/ansible/tasks/internal/optimizations.yml @@ -1,4 +1,5 @@ - name: ensure services are stopped and disabled for first boot nix build + when: stage2 systemd: enabled: no name: '{{ item }}' @@ -10,9 +11,9 @@ - motd-news - vector - salt-minion - when: stage2_nix - name: disable man-db + when: stage2 become: yes file: state: absent @@ -21,4 +22,3 @@ - man-db - popularity-contest - ubuntu-advantage-tools - when: stage2_nix diff --git a/ansible/tasks/internal/postgres-exporter.yml b/ansible/tasks/internal/postgres-exporter.yml index 29dc896e79..03b16474aa 100644 --- a/ansible/tasks/internal/postgres-exporter.yml +++ b/ansible/tasks/internal/postgres-exporter.yml @@ -22,9 +22,9 @@ - name: download postgres exporter get_url: - url: "https://github.com/prometheus-community/postgres_exporter/releases/download/v{{ postgres_exporter_release }}/postgres_exporter-{{ postgres_exporter_release }}.linux-{{ platform }}.tar.gz" + url: "{{ postgres_exporter_artifacts[platform]['url'] }}" + checksum: "{{ postgres_exporter_artifacts[platform]['checksum'] }}" dest: /tmp/postgres_exporter.tar.gz - checksum: "{{ postgres_exporter_release_checksum[platform] }}" timeout: 60 register: download_result until: download_result is succeeded diff --git a/ansible/tasks/internal/supabase-admin-agent.yml b/ansible/tasks/internal/supabase-admin-agent.yml index c5ab847362..1e8b6b8e59 100644 --- a/ansible/tasks/internal/supabase-admin-agent.yml +++ b/ansible/tasks/internal/supabase-admin-agent.yml @@ -59,18 +59,19 @@ mode: "0755" - name: Setting arch (amd64) + when: platform == "amd64" set_fact: arch: "amd64" - when: platform == "amd64" - name: Setting arch (arm) + when: platform == "arm64" set_fact: arch: "arm64" - when: platform == "arm64" - name: Download supabase-admin-agent archive get_url: - url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-agent/v{{ supabase_admin_agent_release }}/supabase-admin-agent-{{ supabase_admin_agent_release }}-linux-{{ arch }}.tar.gz" + url: "{{ supabase_admin_agent_artifacts[platform]['url'] }}" + checksum: "{{ supabase_admin_agent_artifacts[platform]['checksum'] }}" dest: "/tmp/supabase-admin-agent.tar.gz" timeout: 90 register: download_result @@ -90,7 +91,7 @@ - name: supabase-admin-agent - create symlink ansible.builtin.file: path: /opt/supabase-admin-agent/supabase-admin-agent - src: "/opt/supabase-admin-agent/supabase-admin-agent-linux-{{ arch }}" + src: "/opt/supabase-admin-agent/supabase-admin-agent-linux-{{ platform }}" state: link owner: supabase-admin-agent mode: "0755" diff --git a/ansible/tasks/internal/supautils.yml b/ansible/tasks/internal/supautils.yml index dc8866ebf7..0e0d7de284 100644 --- a/ansible/tasks/internal/supautils.yml +++ b/ansible/tasks/internal/supautils.yml @@ -10,8 +10,8 @@ - name: supautils - download latest release get_url: url: "https://github.com/supabase/supautils/archive/refs/tags/v{{ supautils_release }}.tar.gz" - dest: /tmp/supautils-{{ supautils_release }}.tar.gz checksum: "{{ supautils_release_checksum }}" + dest: /tmp/supautils-{{ supautils_release }}.tar.gz timeout: 60 register: download_result until: download_result is succeeded diff --git a/ansible/tasks/setup-envoy.yml b/ansible/tasks/setup-envoy.yml index 7901cd6311..c90ae7be54 100644 --- a/ansible/tasks/setup-envoy.yml +++ b/ansible/tasks/setup-envoy.yml @@ -5,13 +5,12 @@ - name: Envoy - download binary ansible.builtin.get_url: - checksum: "{{ envoy_arm_release_checksum if platform == 'arm64' else envoy_x86_release_checksum }}" + url: "{{ envoy_artifacts[platform]['url'] }}" + checksum: "{{ envoy_artifacts[platform]['checksum'] }}" dest: '/opt/envoy' group: 'envoy' mode: '0700' owner: 'envoy' - # yamllint disable-line rule:line-length - url: "https://github.com/envoyproxy/envoy/releases/download/v{{ envoy_release }}/envoy-{{ envoy_release }}-linux-{{ 'aarch_64' if platform == 'arm64' else 'x86_64' }}" register: download_result until: download_result is succeeded retries: 3 @@ -19,13 +18,12 @@ - name: Envoy - download hot restarter script ansible.builtin.get_url: - checksum: "{{ envoy_hot_restarter_release_checksum }}" + url: "{{ envoy_artifacts['hot_restarter']['url'] }}" + checksum: "{{ envoy_artifacts['hot_restarter']['checksum'] }}" dest: '/opt/envoy-hot-restarter.py' group: 'envoy' mode: '0700' owner: 'envoy' - # yamllint disable-line rule:line-length - url: "https://raw.githubusercontent.com/envoyproxy/envoy/v{{ envoy_release }}/restarter/hot-restarter.py" register: download_result until: download_result is succeeded retries: 3 diff --git a/ansible/tasks/setup-fail2ban.yml b/ansible/tasks/setup-fail2ban.yml index 4c73988661..f4863d2b24 100644 --- a/ansible/tasks/setup-fail2ban.yml +++ b/ansible/tasks/setup-fail2ban.yml @@ -1,7 +1,6 @@ # set default bantime to 1 hour -- name: do nixpkg_mode tasks - when: - - nixpkg_mode +- name: do stage1 tasks + when: not stage2 block: - name: extend the default bantime to an hour become: true diff --git a/ansible/tasks/setup-gotrue.yml b/ansible/tasks/setup-gotrue.yml index 940ce60dcc..58d14f1c1b 100644 --- a/ansible/tasks/setup-gotrue.yml +++ b/ansible/tasks/setup-gotrue.yml @@ -9,20 +9,11 @@ name: 'gotrue' state: 'present' -- name: Setting arch as a fact - ansible.builtin.set_fact: - arch: >- - {%- if platform == 'amd64' -%} - x86 - {%- elif platform == 'arm64' -%} - arm64 - {%- endif -%} - - name: gotrue - download commit archive ansible.builtin.get_url: - checksum: "{{ gotrue_arm_release_checksum if platform == 'arm64' else gotrue_x86_release_checksum }}" + url: "{{ gotrue_artifacts[platform]['url'] }}" + checksum: "{{ gotrue_artifacts[platform]['checksum'] }}" dest: '/tmp/gotrue.tar.gz' - url: "https://github.com/supabase/gotrue/releases/download/v{{ gotrue_release }}/auth-v{{ gotrue_release }}-{{ arch }}.tar.gz" register: download_result until: download_result is succeeded retries: 3 diff --git a/ansible/tasks/setup-kong.yml b/ansible/tasks/setup-kong.yml index 1908481fbb..ec55172de4 100644 --- a/ansible/tasks/setup-kong.yml +++ b/ansible/tasks/setup-kong.yml @@ -14,9 +14,9 @@ - name: Kong - download deb package get_url: - checksum: "{{ kong_deb_checksum[platform] }}" + url: "{{ kong_artifacts[platform]['url'] }}" + checksum: "{{ kong_artifacts[platform]['checksum'] }}" dest: '/tmp/kong.deb' - url: "https://packages.konghq.com/public/gateway-28/deb/ubuntu/pool/{{ kong_release_target }}/main/k/ko/kong_2.8.1/{{ kong_deb[platform] }}" register: download_result until: download_result is succeeded retries: 3 diff --git a/ansible/tasks/setup-migrations.yml b/ansible/tasks/setup-migrations.yml index 9df70a49ae..a5c46a901c 100644 --- a/ansible/tasks/setup-migrations.yml +++ b/ansible/tasks/setup-migrations.yml @@ -1,6 +1,5 @@ -- name: run stage2_nix tasks - when: - - stage2_nix +- name: run stage2 tasks + when: stage2 block: - name: Run migrate.sh script ansible.builtin.command: diff --git a/ansible/tasks/setup-nginx.yml b/ansible/tasks/setup-nginx.yml index 8dda21e7a8..5bf23dbdf4 100644 --- a/ansible/tasks/setup-nginx.yml +++ b/ansible/tasks/setup-nginx.yml @@ -14,9 +14,9 @@ - name: nginx - download source ansible.builtin.get_url: - checksum: "{{ nginx_release_checksum }}" + url: "{{ nginx_artifacts['url'] }}" + checksum: "{{ nginx_artifacts['checksum'] }}" dest: '/tmp/nginx-{{ nginx_release }}.tar.gz' - url: "https://nginx.org/download/nginx-{{ nginx_release }}.tar.gz" register: download_result until: download_result is succeeded retries: 3 diff --git a/ansible/tasks/setup-pgbackrest.yml b/ansible/tasks/setup-pgbackrest.yml index 340fb7d963..6d4eebcdb1 100644 --- a/ansible/tasks/setup-pgbackrest.yml +++ b/ansible/tasks/setup-pgbackrest.yml @@ -1,11 +1,11 @@ - name: Create pgBackRest group + when: not stage2 ansible.builtin.group: name: pgbackrest state: present - when: - - nixpkg_mode - name: Create pgBackRest user + when: not stage2 ansible.builtin.user: comment: pgBackRest user group: pgbackrest @@ -14,8 +14,6 @@ name: pgbackrest shell: /sbin/nologin system: true - when: - - nixpkg_mode - name: Configure sudoers for pgBackRest ansible.builtin.lineinfile: @@ -36,15 +34,6 @@ # that inner sudo fails even though the outer caller is already pgbackrest. - 'pgbackrest ALL=(pgbackrest) NOPASSWD: /var/lib/pgbackrest/.nix-profile/bin/pgbackrest' -- name: Install pgBackRest - ansible.builtin.shell: | - sudo -u pgbackrest bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#pg-backrest" - become: true - become_user: pgbackrest - changed_when: true - when: - - stage2_nix - - name: Create needed directories for pgBackRest ansible.legacy.file: group: postgres @@ -62,8 +51,6 @@ - {dir: /var/log/pgbackrest} loop_control: loop_var: backrest_dir - when: - - nixpkg_mode or stage2_nix # supabase-admin-agent opens these log files with O_APPEND|O_WRONLY (no # O_CREATE). They must exist before SAA runs; a missing file causes the @@ -82,8 +69,6 @@ - /var/log/pgbackrest/saa-pgb.log - /var/log/pgbackrest/wal-push.log - /var/log/pgbackrest/wal-fetch.log - when: - - nixpkg_mode or stage2_nix - name: Symlink pgbackrest.conf ansible.legacy.file: @@ -92,24 +77,6 @@ src: /etc/pgbackrest.conf state: link -- name: Move pgBackRest files to /etc/pgbackrest - ansible.legacy.copy: - group: postgres - dest: "/etc/pgbackrest/{{ conf_item['path'] }}/{{ conf_item['name'] }}" - mode: '0640' - owner: pgbackrest - src: "files/pgbackrest_config/{{ conf_item['name'] }}" - loop: - - {name: computed_globals.conf, path: conf.d} - - {name: pgbackrest.conf, path: ''} - - {name: repo1_async.conf, path: conf.d} - - {name: repo1_encrypted.conf, path: conf.d} - - {name: repo1.conf, path: conf.d} - loop_control: - loop_var: conf_item - when: - - stage2_nix - - name: Create pgBackRest wrapper script ansible.builtin.copy: content: | @@ -149,3 +116,28 @@ group: 'root' mode: '0755' owner: 'root' + +- name: Install pgBackRest + when: stage2 + ansible.builtin.shell: | + sudo -u pgbackrest bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#pg-backrest" + become: true + become_user: pgbackrest + changed_when: true + +- name: Move pgBackRest files to /etc/pgbackrest + when: stage2 + ansible.legacy.copy: + group: postgres + dest: "/etc/pgbackrest/{{ conf_item['path'] }}/{{ conf_item['name'] }}" + mode: '0640' + owner: pgbackrest + src: "files/pgbackrest_config/{{ conf_item['name'] }}" + loop: + - {name: computed_globals.conf, path: conf.d} + - {name: pgbackrest.conf, path: ''} + - {name: repo1_async.conf, path: conf.d} + - {name: repo1_encrypted.conf, path: conf.d} + - {name: repo1.conf, path: conf.d} + loop_control: + loop_var: conf_item diff --git a/ansible/tasks/setup-pgbouncer.yml b/ansible/tasks/setup-pgbouncer.yml index 16cdfbb3b2..2a1b94d5c8 100644 --- a/ansible/tasks/setup-pgbouncer.yml +++ b/ansible/tasks/setup-pgbouncer.yml @@ -13,10 +13,10 @@ - name: PgBouncer - download latest release ansible.builtin.get_url: - checksum: "{{ pgbouncer_release_checksum }}" + url: "{{ pgbouncer_artifacts['url'] }}" + checksum: "{{ pgbouncer_artifacts['checksum'] }}" dest: "/tmp/pgbouncer-{{ pgbouncer_release }}.tar.gz" timeout: 60 - url: "https://www.pgbouncer.org/downloads/files/{{ pgbouncer_release }}/pgbouncer-{{ pgbouncer_release }}.tar.gz" register: download_result until: download_result is succeeded retries: 3 @@ -69,6 +69,7 @@ - { mode: '0775', dir: '/etc/pgbouncer-custom' } - name: create PgBouncer placeholder config files + when: not stage2 ansible.builtin.file: group: 'pgbouncer' mode: '0664' @@ -81,7 +82,6 @@ - 'custom-overrides.ini' - 'generated-optimizations.ini' - 'ssl-config.ini' - when: nixpkg_mode - name: PgBouncer - adjust pgbouncer.ini ansible.builtin.copy: diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index be6bb8ad15..c4d065451b 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -1,39 +1,5 @@ -- name: execute stage2_nix tasks - when: - - stage2_nix - block: - - name: install locales - ansible.builtin.apt: - name: 'locales' - state: 'present' - become: true - - - name: configure locales - ansible.builtin.lineinfile: - create: true - line: "{{ locale_item }}.UTF-8 UTF-8" - path: '/etc/locale.gen' - state: 'present' - become: true - loop: - - 'C' - - 'en_US' - loop_control: - loop_var: 'locale_item' - - - name: locale-gen - ansible.builtin.command: - cmd: 'locale-gen' - become: true - - - name: update-locale - ansible.builtin.command: - cmd: 'update-locale' - become: true - -- name: execute nixpkg_mode tasks - when: - - nixpkg_mode +- name: execute stage1 tasks + when: not stage2 block: # the old method of installing from debian creates this group, but we must create it explicitly # for the nix built version @@ -60,10 +26,6 @@ system: true become: yes -- name: execute nixpkg_mode tasks - when: - - nixpkg_mode - block: - name: Create relevant directories ansible.builtin.file: group: 'postgres' @@ -150,33 +112,14 @@ is_psql_15: "{{ psql_version in ['psql_15'] }}" - name: create placeholder pam config + when: not is_psql_15 file: path: '/etc/pam.d/postgresql' state: touch owner: postgres group: postgres mode: 0664 - when: not is_psql_15 - -# Install extensions before init -- name: Install Postgres extensions - ansible.builtin.import_tasks: - file: 'tasks/setup-docker.yml' - when: - - stage2_nix -#stage 2 postgres tasks -- name: stage2 postgres tasks - ansible.builtin.import_tasks: - file: 'tasks/stage2-setup-postgres.yml' - when: - - stage2_nix - -# init DB -- name: init the db when nixpkg_mode - when: - - nixpkg_mode - block: - name: Create directory on data volume ansible.builtin.file: group: 'postgres' @@ -193,32 +136,66 @@ src: '/data/pgdata' state: 'link' -- name: Make sure .bashrc exists - ansible.builtin.file: - group: 'postgres' - owner: 'postgres' - path: '/var/lib/postgresql/.bashrc' - state: 'touch' - when: - - nixpkg_mode + - name: Make sure .bashrc exists + ansible.builtin.file: + group: 'postgres' + owner: 'postgres' + path: '/var/lib/postgresql/.bashrc' + state: 'touch' -- name: Ensure /data configured for production runtime - when: - - stage2_nix and (qemu_mode is not defined) - ansible.posix.mount: - src: "UUID={{ ansible_mounts | selectattr('mount', 'equalto', '/data') | map(attribute='uuid') | first }}" - path: /data - fstype: ext4 - opts: defaults,discard,nofail,x-systemd.device-timeout=5s - dump: "0" - passno: "2" - state: mounted - -- name: Run initdb (stage2_nix) - when: - - stage2_nix +- name: execute stage2 tasks + when: stage2 block: - - name: Initialize the database stage2_nix (NOT PG17 NOR PG17-OrioleDB) + - name: install locales + ansible.builtin.apt: + name: 'locales' + state: 'present' + become: true + + - name: configure locales + ansible.builtin.lineinfile: + create: true + line: "{{ locale_item }}.UTF-8 UTF-8" + path: '/etc/locale.gen' + state: 'present' + become: true + loop: + - 'C' + - 'en_US' + loop_control: + loop_var: 'locale_item' + + - name: locale-gen + ansible.builtin.command: + cmd: 'locale-gen' + become: true + + - name: update-locale + ansible.builtin.command: + cmd: 'update-locale' + become: true + + - name: Install Postgres extensions + ansible.builtin.import_tasks: + file: 'tasks/setup-supautils.yml' + + - name: stage2 postgres tasks + ansible.builtin.import_tasks: + file: 'tasks/stage2-setup-postgres.yml' + + - name: Ensure /data configured for production runtime + when: stage2 and not qemu + ansible.posix.mount: + src: "UUID={{ ansible_mounts | selectattr('mount', 'equalto', '/data') | map(attribute='uuid') | first }}" + path: /data + fstype: ext4 + opts: defaults,discard,nofail,x-systemd.device-timeout=5s + dump: "0" + passno: "2" + state: mounted + + - name: Initialize the database stage2 (NOT PG17 NOR PG17-OrioleDB) + when: psql_version not in ['psql_17', 'psql_orioledb-17'] become: true become_user: 'postgres' ansible.builtin.command: @@ -231,9 +208,9 @@ LOCALE_ARCHIVE: /usr/lib/locale/locale-archive vars: ansible_command_timeout: 60 - when: psql_version not in ['psql_17', 'psql_orioledb-17'] - - name: Initialize the database stage2_nix (PG17 OR PG17-OrioleDB) + - name: Initialize the database stage2 (PG17 OR PG17-OrioleDB) + when: psql_version in ['psql_17', 'psql_orioledb-17'] become: true become_user: 'postgres' ansible.builtin.command: @@ -246,7 +223,6 @@ LOCALE_ARCHIVE: /usr/lib/locale/locale-archive vars: ansible_command_timeout: 60 - when: psql_version in ['psql_17', 'psql_orioledb-17'] - name: copy PG apparmor profile ansible.builtin.copy: @@ -269,6 +245,7 @@ cmd: '/usr/sbin/aa-enforce sbpostgres' - name: copy PG and optimizations systemd units + when: stage2 ansible.builtin.template: dest: "/etc/systemd/system/{{ systemd_svc_item | basename }}" src: "files/{{ systemd_svc_item }}" @@ -277,8 +254,6 @@ - 'postgresql_config/postgresql.service' loop_control: loop_var: 'systemd_svc_item' - when: - - stage2_nix - name: initialize pg required state become: true @@ -288,10 +263,11 @@ path: '/run/postgresql' state: 'directory' when: - - stage2_nix - - qemu_mode is defined + - stage2 + - qemu - name: Restart Postgres Database without Systemd + when: stage2 become: true become_user: 'postgres' ansible.builtin.command: @@ -302,12 +278,9 @@ LC_ALL: en_US.UTF-8 LC_CTYPE: en_US.UTF-8 LOCALE_ARCHIVE: /usr/lib/locale/locale-archive - when: - - stage2_nix - # Reload -- name: Create a SystemD override dir for PostgreSQL +- name: Create a Systemd override dir for PostgreSQL ansible.builtin.file: group: 'root' mode: '0755' @@ -343,16 +316,16 @@ section: 'Unit' state: 'present' value: 'tuned.service' + - name: System - systemd reload + when: stage2 ansible.builtin.systemd_service: daemon_reload: true enabled: true name: 'postgresql' - when: - - stage2_nix - - name: Add lang and locale items to .bashrc + when: not stage2 ansible.builtin.lineinfile: create: true dest: '/var/lib/postgresql/.bashrc' @@ -367,5 +340,3 @@ - 'export LC_CTYPE="en_US.UTF-8"' loop_control: loop_var: 'lang_item' - when: - - nixpkg_mode diff --git a/ansible/tasks/setup-postgrest.yml b/ansible/tasks/setup-postgrest.yml index 2fd001fa5b..c1134aeb9e 100644 --- a/ansible/tasks/setup-postgrest.yml +++ b/ansible/tasks/setup-postgrest.yml @@ -5,12 +5,12 @@ - name: PostgREST - add Postgres PPA gpg key ansible.builtin.get_url: + url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' + checksum: sha256:0144068502a1eddd2a0280ede10ef607d1ec592ce819940991203941564e8e76 dest: /etc/apt/trusted.gpg.d/ppdg.asc force: true mode: '0644' - url: 'https://www.postgresql.org/media/keys/ACCC4CF8.asc' timeout: 60 - validate_certs: true register: pgdg_key_download retries: 6 delay: 2 @@ -57,21 +57,14 @@ - name: PostgREST - download ubuntu binary archive (arm) ansible.builtin.get_url: - checksum: "{{ postgrest_arm_release_checksum if platform == 'arm64' else postgrest_x86_release_checksum }}" + url: "{{ postgrest_artifacts[platform]['url'] }}" + checksum: "{{ postgrest_artifacts[platform]['checksum'] }}" dest: '/tmp/postgrest.tar.xz' timeout: 60 - url: "https://github.com/PostgREST/postgrest/releases/download/v{{ postgrest_release }}/postgrest-v{{ postgrest_release }}-{{ download_binary }}.tar.xz" register: download_result until: download_result is succeeded retries: 3 delay: 2 - vars: - download_binary: >- - {%- if platform == "arm64" -%} - ubuntu-aarch64 - {%- elif platform == "amd64" -%} - linux-static-x86-64 - {%- endif -%} - name: PostgREST - unpack archive in /opt ansible.builtin.unarchive: diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml index 78a376efb4..2318c4d609 100644 --- a/ansible/tasks/setup-supabase-internal.yml +++ b/ansible/tasks/setup-supabase-internal.yml @@ -4,7 +4,8 @@ - name: download Vector package ansible.builtin.get_url: - url: "{{ vector_x86_deb if platform == 'amd64' else vector_arm_deb }}" + url: "{{ vector_artifacts[platform]['url'] }}" + checksum: "{{ vector_artifacts[platform]['checksum'] }}" dest: /tmp/vector.deb timeout: 120 become: true diff --git a/ansible/tasks/setup-docker.yml b/ansible/tasks/setup-supautils.yml similarity index 95% rename from ansible/tasks/setup-docker.yml rename to ansible/tasks/setup-supautils.yml index 88d093af32..d9520191f7 100644 --- a/ansible/tasks/setup-docker.yml +++ b/ansible/tasks/setup-supautils.yml @@ -1,8 +1,6 @@ -- name: stage2_nix actions - when: - - stage2_nix +- name: Setup Supautils + when: stage2 block: - # supautils - name: supautils - add supautils to session_preload_libraries ansible.builtin.replace: path: '/etc/postgresql/postgresql.conf' diff --git a/ansible/tasks/setup-system.yml b/ansible/tasks/setup-system.yml index 8059ecd9c7..6777b72488 100644 --- a/ansible/tasks/setup-system.yml +++ b/ansible/tasks/setup-system.yml @@ -1,5 +1,5 @@ -- name: Execute tasks when nixpkg_mode - when: nixpkg_mode +- name: Execute stage1 tasks + when: not stage2 block: - name: System - apt update and apt upgrade ansible.builtin.apt: @@ -76,15 +76,6 @@ owner: 'root' group: 'root' -- name: Set the platform arch as a fact - ansible.builtin.set_fact: - platform: "{{ 'amd64' if ansible_facts['architecture'] == 'x86_64' else 'arm64' }}" - tags: - - update - - update-only - when: - - nixpkg_mode or stage2_nix - - name: Disable algif_aead module autoload for Copy Fail mitigation ansible.builtin.copy: content: | @@ -93,20 +84,15 @@ mode: '0644' owner: root group: root - when: - - nixpkg_mode or stage2_nix - name: Unload algif_aead module if currently loaded ansible.builtin.command: cmd: rmmod algif_aead changed_when: false failed_when: false - when: - - nixpkg_mode or stage2_nix -- name: Execute more tasks when nixpkg_mode - when: - - nixpkg_mode +- name: Execute more tasks when stage1 + when: not stage2 block: - name: Custom systemd overrides for resolved ansible.builtin.copy: @@ -165,6 +151,7 @@ group: 'root' - name: set hosts file + when: stage2 ansible.builtin.copy: content: | 127.0.0.1 localhost @@ -173,8 +160,6 @@ mode: '0644' owner: 'root' group: 'root' - when: - - stage2_nix - name: Configure Apt ansible.builtin.copy: diff --git a/ansible/tasks/setup-tuned.yml b/ansible/tasks/setup-tuned.yml index 0677d1942a..6b5c4595ab 100644 --- a/ansible/tasks/setup-tuned.yml +++ b/ansible/tasks/setup-tuned.yml @@ -1,6 +1,4 @@ - name: 'tuned - Install and configure tuned' # noqa: name[casing] - when: - - (stage2_nix or nixpkg_mode) block: - name: 'tuned - Install tuned' # noqa: name[casing] ansible.builtin.apt: @@ -71,6 +69,7 @@ become: true - name: 'tuned - Configure Intel CPUs for maximum performance' # noqa: name[casing] + when: ansible_facts['processor'][0] is search('GenuineIntel', ignorecase=True) community.general.ini_file: create: true group: 'root' @@ -91,8 +90,6 @@ value: '100' loop_control: loop_var: 'intel_item' - when: - - ansible_facts['processor'][0] is search('GenuineIntel', ignorecase=True) - name: 'tuned - Set sysctl parameters' # noqa: name[casing] community.general.ini_file: @@ -175,6 +172,7 @@ loop_var: 'sysctl_item' - name: 'tuned - Set kernel.panic=10' # noqa: name[casing] + when: not stage2 community.general.ini_file: create: true group: 'root' @@ -186,12 +184,9 @@ state: 'present' value: '10' become: true - when: - - nixpkg_mode - name: 'tuned - Enable zswap if swap is present' # noqa: name[casing] - when: - - ansible_facts['swaptotal_mb'] > 0 + when: ansible_facts['swaptotal_mb'] > 0 block: - name: 'tuned - Decrease the kernel swappiness' # noqa: name[casing] community.general.ini_file: @@ -234,13 +229,12 @@ daemon_reload: true enabled: true name: 'tuned' - state: "{{ 'restarted' if stage2_nix else 'stopped' }}" + state: "{{ 'restarted' if stage2 else 'stopped' }}" become: true - name: 'tuned - Activate the PostgreSQL tuned profile' # noqa: name[casing] + when: stage2 ansible.builtin.command: cmd: 'tuned-adm profile postgresql' become: true changed_when: false - when: - - stage2_nix diff --git a/ansible/tasks/setup-wal-g.yml b/ansible/tasks/setup-wal-g.yml index 0d923cd05d..d7311beb7a 100644 --- a/ansible/tasks/setup-wal-g.yml +++ b/ansible/tasks/setup-wal-g.yml @@ -1,6 +1,5 @@ -- name: Execute wal-g things when nixpkg_mode - when: - - nixpkg_mode +- name: Execute wal-g things when stage1 + when: not stage2 block: - name: Create wal-g group ansible.builtin.group: @@ -24,9 +23,8 @@ path: '/etc/wal-g' state: 'directory' -- name: Execute wal-g things when stage2_nix - when: - - stage2_nix +- name: Execute wal-g things when stage2 + when: stage2 block: - name: Install wal-g 2 from nix binary cache ansible.builtin.shell: diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index d935d24473..1a2bd9d9e6 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -6,11 +6,9 @@ is_psql_17: "{{ psql_version in ['psql_17'] }}" is_psql_15: "{{ psql_version in ['psql_15'] }}" - - name: Execute tasks when (is_psql_oriole or is_psql_17) and stage2_nix + - name: Execute tasks when (is_psql_oriole or is_psql_17) and stage2 + when: stage2 and (is_psql_oriole or is_psql_17) become: true - when: - - (is_psql_oriole or is_psql_17) - - stage2_nix block: - name: Remove specified extensions from postgresql.conf if orioledb-17 or 17 build ansible.builtin.replace: @@ -34,11 +32,9 @@ regexp: 'db_user_namespace\ =\ off' replace: '#db_user_namespace\ =\ off' - - name: Execute things when is_psql_oriole and stage2_nix + - name: Execute things when is_psql_oriole and stage2 + when: stage2 and is_psql_oriole become: true - when: - - is_psql_oriole - - stage2_nix block: - name: Append orioledb to shared_preload_libraries append within closing quote ansible.builtin.replace: @@ -57,10 +53,9 @@ line: 'ORIOLEDB_ENABLED=true' path: '/etc/environment' -- name: Execute things when stage2_nix +- name: Execute things when stage2 + when: stage2 become: true - when: - - stage2_nix block: - name: Install packages from nix binary cache ansible.builtin.shell: | @@ -130,9 +125,7 @@ is_psql_15: "{{ psql_version == 'psql_15' }}" - name: Install gatekeeper if not pg15 - when: - - stage2_nix - - not is_psql_15 + when: stage2 and not is_psql_15 block: - name: Install gatekeeper from nix binary cache become: yes diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 1733e6ca6d..3acd25cefa 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -1,6 +1,5 @@ -- name: Execute tasks when stage2_nix - when: - - stage2_nix +- name: Execute tasks when stage2 + when: stage2 block: - name: Make a backup of the /etc/postgresql/postgresql.conf file ansible.builtin.copy: @@ -39,10 +38,6 @@ loop_var: 'regx' register: 'pgconf' -- name: Execute tasks when stage2_nix - when: - - stage2_nix - block: - name: Restart Postgres Database in stage 2 to load all extensions ansible.builtin.command: cmd: "/usr/lib/postgresql/bin/pg_ctl --pgdata /var/lib/postgresql/data --mode fast --options '-c config_file=/etc/postgresql/postgresql.conf' {{ ctlcmd }}" @@ -60,10 +55,6 @@ loop_control: loop_var: 'ctlcmd' -- name: Execute tasks when stage2_nix - when: - - stage2_nix - block: - name: Re-enable PG Sodium references in /etc/postgresql/postgresql.conf ansible.builtin.command: cmd: mv /etc/postgresql/postgresql.conf.bak /etc/postgresql/postgresql.conf diff --git a/ansible/vars.yml b/ansible/vars.yml index c540b0441a..8f6507b07b 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -2,68 +2,110 @@ supabase_internal: true ebssurrogate_mode: true async_mode: true - postgres_major: - "15" - "17" - orioledb-17 - # Full version strings for each major version postgres_release: postgresorioledb-17: "17.9.0.009-orioledb" postgres17: "17.6.1.156" postgres15: "15.14.1.156" - # Non Postgres Extensions pgbouncer_release: 1.25.1 -pgbouncer_release_checksum: sha256:6e566ae92fe3ef7f6a1b9e26d6049f7d7ca39c40e29e7b38f6d5500ae15d8465 - +pgbouncer_artifacts: + url: "https://www.pgbouncer.org/downloads/files/{{ pgbouncer_release }}/pgbouncer-{{ pgbouncer_release }}.tar.gz" + checksum: sha256:6e566ae92fe3ef7f6a1b9e26d6049f7d7ca39c40e29e7b38f6d5500ae15d8465 # The checksum can be found under "Assets", in the GitHub release page for each version. # The binaries used are: ubuntu-aarch64 and linux-static. # https://github.com/PostgREST/postgrest/releases postgrest_release: "14.15" # keep this as a string, otherwise gets treated as yaml float which will likely only lead to tears -postgrest_arm_release_checksum: sha256:1eb007298c1536ba865e741da7eece6fba6db3da904c599abd15d9c3debe6c2f -postgrest_x86_release_checksum: sha256:4e78c7f065a6c36f350c7177f5fa9bb77ea380c67b8bf40f2fc9130d857678dc - +postgrest_artifacts: + amd64: + url: "https://github.com/PostgREST/postgrest/releases/download/v{{ postgrest_release }}/postgrest-v{{ postgrest_release }}-linux-static-x86-64.tar.xz" + checksum: sha256:4e78c7f065a6c36f350c7177f5fa9bb77ea380c67b8bf40f2fc9130d857678dc + arm64: + url: "https://github.com/PostgREST/postgrest/releases/download/v{{ postgrest_release }}/postgrest-v{{ postgrest_release }}-ubuntu-aarch64.tar.xz" + checksum: sha256:1eb007298c1536ba865e741da7eece6fba6db3da904c599abd15d9c3debe6c2f gotrue_release: 2.190.0 -gotrue_release_checksum: sha1:7ceab0d91ebc68792afb58d07cfb73e1505de882 -gotrue_arm_release_checksum: sha1:7ceab0d91ebc68792afb58d07cfb73e1505de882 -gotrue_x86_release_checksum: sha1:56725ea31e32601dd5861f2e6f5d1e3cfd72299b - +gotrue_artifacts: + amd64: + url: "https://github.com/supabase/gotrue/releases/download/v{{ gotrue_release }}/auth-v{{ gotrue_release }}-x86.tar.gz" + checksum: sha256:dfe8e2dee872747f60fe10ecec6ee8d368ee4d7eef3acd5af95aa88a2e7e6141 + arm64: + url: "https://github.com/supabase/gotrue/releases/download/v{{ gotrue_release }}/auth-v{{ gotrue_release }}-arm64.tar.gz" + checksum: sha256:547b80c455f2bbf01a624d0e3bc98c31237ca479d4aa0d236d40678370e34a69 aws_cli_release: 2.23.11 - +aws_cli_artifacts: + amd64: + url: "https://awscli.amazonaws.com/awscli-exe-linux-x86_64-{{ aws_cli_release }}.zip" + checksum: sha256:d9cad44c58bbf5f43cfccd291d688d5297356608aeaa487ce65c4ac81251f4c7 + arm64: + url: "https://awscli.amazonaws.com/awscli-exe-linux-aarch64-{{ aws_cli_release }}.zip" + checksum: sha256:21e74a95c28ba2c9a55e6af215484ab4028500c8dc98b7188a2c9854ba116e8f salt_version: "3007.14" # keep this as a string, otherwise gets treated as yaml float which will likely only lead to tears - -golang_version: 1.22.11 -golang_version_checksum: - arm64: sha256:0fc88d966d33896384fbde56e9a8d80a305dc17a9f48f1832e061724b1719991 - amd64: sha256:9ebfcab26801fa4cf0627c6439db7a4da4d3c6766142a3dd83508240e4f21031 - envoy_release: 1.28.0 -envoy_arm_release_checksum: sha256:eb930e32ab5555643e09d11d490e392d0a790c5a80eb0b0ebacb1046bdbb114d -envoy_x86_release_checksum: sha256:2639563ce9bc09b1ee6bd4731760b596c8ed1e474cdd83cc9df9364516e14367 -envoy_hot_restarter_release_checksum: sha1:6d43b89d266fb2427a4b51756b649883b0617eda - +envoy_artifacts: + amd64: + url: "https://github.com/envoyproxy/envoy/releases/download/v{{ envoy_release }}/envoy-{{ envoy_release }}-linux-x86_64" + checksum: sha256:2639563ce9bc09b1ee6bd4731760b596c8ed1e474cdd83cc9df9364516e14367 + arm64: + url: "https://github.com/envoyproxy/envoy/releases/download/v{{ envoy_release }}/envoy-{{ envoy_release }}-linux-aarch_64" + checksum: sha256:eb930e32ab5555643e09d11d490e392d0a790c5a80eb0b0ebacb1046bdbb114d + hot_restarter: + url: "https://raw.githubusercontent.com/envoyproxy/envoy/v{{ envoy_release }}/restarter/hot-restarter.py" + checksum: sha256:3a2f9b1056771787eb0304dceb7e42013c6cf1c8e8281ff63f573fa5f258c709 +kong_release: 2.8.1 kong_release_target: focal -kong_deb: - arm64: kong_2.8.1_arm64.deb - amd64: kong_2.8.1_amd64.deb -kong_deb_checksum: - arm64: sha1:2086f6ccf8454fe64435252fea4d29d736d7ec61 - amd64: sha1:7346d558a467f05d9ed63675b59f974300647e9f - +kong_artifacts: + amd64: + url: "https://packages.konghq.com/public/gateway-28/deb/ubuntu/pool/{{ kong_release_target }}/main/k/ko/kong_{{kong_release}}/kong_{{ kong_release }}_amd64.deb" + checksum: sha256:10d12d23e5890414d666663094d51a42de41f8a9806fbc0baaf9ac4d37794361 + arm64: + url: "https://packages.konghq.com/public/gateway-28/deb/ubuntu/pool/{{ kong_release_target }}/main/k/ko/kong_{{kong_release}}/kong_{{ kong_release }}_arm64.deb" + checksum: sha256:61c13219ef64dac9aeae5ae775411e8cfcd406f068cf3e75d463f916ae6513cb nginx_release: 1.22.0 -nginx_release_checksum: sha1:419efb77b80f165666e2ee406ad8ae9b845aba93 - -postgres_exporter_release: "0.15.0" -postgres_exporter_release_checksum: - arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0 - amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84 - -adminapi_release: "0.112.1" -adminmgr_release: "0.43.6" -supabase_admin_agent_release: 1.14.4 +nginx_artifacts: + url: "https://nginx.org/download/nginx-{{ nginx_release }}.tar.gz" + checksum: sha256:b33d569a6f11a01433a57ce17e83935e953ad4dc77cdd4d40f896c88ac26eb53 +postgres_exporter_release: 0.15.0 +postgres_exporter_artifacts: + amd64: + url: "https://github.com/prometheus-community/postgres_exporter/releases/download/v{{ postgres_exporter_release }}/postgres_exporter-{{ postgres_exporter_release }}.linux-amd64.tar.gz" + checksum: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84 + arm64: + url: "https://github.com/prometheus-community/postgres_exporter/releases/download/v{{ postgres_exporter_release }}/postgres_exporter-{{ postgres_exporter_release }}.linux-arm64.tar.gz" + checksum: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0 +adminapi_release: 0.112.1 +adminapi_artifacts: + amd64: + url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-api/v{{ adminapi_release }}/supabase-admin-api_{{ adminapi_release }}_linux_amd64.tar.gz" + checksum: sha256:e7b94f29a7387f336846a714e3b2202c2579027efd30a3d919b41b02fd288569 + arm64: + url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-api/v{{ adminapi_release }}/supabase-admin-api_{{ adminapi_release }}_linux_arm64.tar.gz" + checksum: sha256:893f35ff3fcfc6088127d96998325be34bc4dc7ba59f74d38f47790beceebaf5 +adminmgr_release: 0.43.6 +adminmgr_artifacts: + amd64: + url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/admin-mgr/v{{ adminmgr_release }}/admin-mgr_{{ adminmgr_release }}_linux_amd64.tar.gz" + checksum: sha256:199c44e2c53e8cc73029f6b6bff64e47491cba5d59c02427facee3112e54f5b0 + arm64: + url: "https://supabase-public-artifacts-bucket.s3.amazonaws.com/admin-mgr/v{{ adminmgr_release }}/admin-mgr_{{ adminmgr_release }}_linux_arm64.tar.gz" + checksum: sha256:0e649f0714ca47735d0e2a27ae740abd43d64b9b472b6cf113393390b2c1a694 supabase_admin_agent_splay: 30s - -vector_x86_deb: https://packages.timber.io/vector/0.48.X/vector_0.48.0-1_amd64.deb -vector_arm_deb: https://packages.timber.io/vector/0.48.X/vector_0.48.0-1_arm64.deb +supabase_admin_agent_release: 1.14.4 +supabase_admin_agent_artifacts: + amd64: + url: https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-agent/v{{ supabase_admin_agent_release }}/supabase-admin-agent-{{ supabase_admin_agent_release }}-linux-amd64.tar.gz + checksum: sha256:7b0c03c3d2401f2ac41d82fad1d44a917c6f7a81ea8f8090f5c42c802b83ba21 + arm64: + url: https://supabase-public-artifacts-bucket.s3.amazonaws.com/supabase-admin-agent/v{{ supabase_admin_agent_release }}/supabase-admin-agent-{{ supabase_admin_agent_release }}-linux-arm64.tar.gz + checksum: sha256:7db4e9849d7d6131c79cc9bba21a7bb2e0949b59367b390f58d4dc526ce766c3 +vector_release: 0.48.0-1 +vector_artifacts: + amd64: + url: "https://packages.timber.io/vector/{{ vector_release.split('.')[:2] | join('.') }}.X/vector_{{ vector_release }}_amd64.deb" + checksum: sha256:a898f8e44b58ad9b3a87f877dfed160fa469edab1e4cbae5b7700d0ad28a444b + arm64: + url: "https://packages.timber.io/vector/{{ vector_release.split('.')[:2] | join('.') }}.X/vector_{{ vector_release }}_arm64.deb" + checksum: sha256:c52483bbd1aba387e8d83eded6fce0c1de31eb7eac810ad4d780895a887da6b1 diff --git a/ebssurrogate/scripts/qemu-bootstrap-nix.sh b/ebssurrogate/scripts/qemu-bootstrap-nix.sh index 944d34d978..0c432cb696 100755 --- a/ebssurrogate/scripts/qemu-bootstrap-nix.sh +++ b/ebssurrogate/scripts/qemu-bootstrap-nix.sh @@ -50,7 +50,7 @@ callbacks_enabled = timer, profile_tasks, profile_roles EOF # Run Ansible playbook export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp - ansible-playbook ./ansible/playbook.yml --extra-vars '{"nixpkg_mode": true, "stage2_nix": false, "qemu_mode": true}' \ + ansible-playbook ./ansible/playbook.yml --extra-vars '{"stage2": false, "qemu": true}' \ --extra-vars "postgresql_version=postgresql_${POSTGRES_MAJOR_VERSION}" \ --extra-vars "postgresql_major_version=${POSTGRES_MAJOR_VERSION}" \ --extra-vars "postgresql_major=${POSTGRES_MAJOR_VERSION}" \ @@ -116,7 +116,7 @@ EOF # Run Ansible playbook export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/tmp ansible-playbook ./ansible/playbook.yml \ - --extra-vars '{"nixpkg_mode": false, "stage2_nix": true, "qemu_mode": true}' \ + --extra-vars '{"stage2": true, "qemu": true}' \ --extra-vars "git_commit_sha=${GIT_SHA}" \ --extra-vars "postgresql_version=postgresql_${POSTGRES_MAJOR_VERSION}" \ --extra-vars "postgresql_major_version=${POSTGRES_MAJOR_VERSION}" \ diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 947fd86077..a89f7daf50 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -341,7 +341,7 @@ EOF #export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml \ - --extra-vars '{"nixpkg_mode": true, "stage2_nix": false} ' \ + --extra-vars '{"stage2":false, "qemu":true} ' \ --extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" \ $ARGS } diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index f1f4b9e6f5..8a69f7156a 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -52,7 +52,7 @@ EOF # Run Ansible playbook export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/tmp ansible-playbook /tmp/ansible-playbook/ansible/playbook.yml \ - --extra-vars '{"nixpkg_mode": false, "stage2_nix": true}' \ + --extra-vars '{"stage2":true, "qemu":false}' \ --extra-vars "git_commit_sha=${GIT_SHA}" \ --extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" \ --extra-vars "postgresql_version=postgresql_${POSTGRES_MAJOR_VERSION}" \ diff --git a/scripts/update-ansible-artifact.sh b/scripts/update-ansible-artifact.sh new file mode 100755 index 0000000000..a53cd2619c --- /dev/null +++ b/scripts/update-ansible-artifact.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# == 1)) && [[ $1 == verify ]]; then + operation=verify +elif (($# == 2)); then + operation=update +else + echo "Usage: $(basename "$0") " >&2 + echo "Usage: $(basename "$0") verify" >&2 + exit 1 +fi + +root=$(git rev-parse --show-toplevel) +vars=$root/ansible/vars.yml + +if [[ $operation == verify ]]; then + failures=0 + artifacts=0 + while IFS=$'\t' read -r path url expected; do + ((++artifacts)) + if ! checksum=$(curl --fail --location --silent --show-error "$url" | sha256sum); then + echo "FAIL $path: download failed" >&2 + ((++failures)) + continue + fi + + actual=${checksum%% *} + if [[ $expected == sha256:$actual ]]; then + echo "OK $path" >&2 + else + echo "FAIL $path: expected $expected, got sha256:$actual" >&2 + ((++failures)) + fi + done < <( + jinja2 --format=yaml "$vars" "$vars" | + yq -r '.. | select(tag == "!!map" and has("url") and has("checksum")) | [(path | join(".")), .url, .checksum] | @tsv' | + sort + ) + + if ((artifacts == 0)); then + echo "error: no artifacts found" >&2 + exit 1 + fi + if ((failures > 0)); then + echo "$failures of $artifacts artifacts failed verification" >&2 + exit 1 + fi + exit 0 +fi + +tool=$1 +version=$2 +release=${tool}_release +artifacts=${tool}_artifacts + +if ! yq 'keys[]' "$vars" | grep -x "$release" >/dev/null; then + echo "error: uknown tool $tool, missing $release" >&2 + exit 1 +fi + +if ! yq 'keys[]' "$vars" | grep -x "$artifacts" >/dev/null; then + echo "error: uknown tool $tool, missing $artifacts" >&2 + exit 1 +fi + +workdir=$(mktemp -d "$root-ansible-vars-update-XXXXXX") +trap 'rm -rf "$workdir"' EXIT +cd "$workdir" + +VERSION=$version yq ".$release = strenv(VERSION)" "$vars" >vars.yml +jinja2 --format=yaml vars.yml vars.yml >rendered.yml + +urls=() +checksum_paths=() +if yq -e ".$artifacts.url" rendered.yml &>/dev/null; then + urls+=("$(yq -r ".$artifacts.url" rendered.yml)") + checksum_paths+=(".$artifacts.checksum") +else + mapfile -t artifact_keys < <(yq -r ".$artifacts | keys | .[]" rendered.yml) + for key in "${artifact_keys[@]}"; do + urls+=("$(yq -r ".$artifacts[\"$key\"].url" rendered.yml)") + checksum_paths+=(".$artifacts[\"$key\"].checksum") + done +fi + +((${#urls[@]} > 0 && ${#urls[@]} == ${#checksum_paths[@]})) || { + echo "error: no artifact URLs found in $artifacts" >&2 + exit 1 +} + +checksums=() +for url in "${urls[@]}"; do + echo "Downloading $url" >&2 + checksum=$(curl --fail --location --silent --show-error "$url" | sha256sum) + checksums+=("${checksum%% *}") +done + +for index in "${!checksums[@]}"; do + CHECKSUM=sha256:${checksums[index]} yq -i "${checksum_paths[index]} = strenv(CHECKSUM)" vars.yml +done + +if ! cmp --silent vars.yml "$vars"; then + mv vars.yml "$vars" + echo "ansible/vars.yml updated $tool -> $version" >&2 +else + echo "ansible/vars.yml is already up to date" >&2 +fi