feat(wal-g): install wal-g 3.0.8 alongside v2 (INDATA-904)

ansible/files/postgresql_config/sbpostgres_apparmor

@@ -66,6 +66,7 @@ profile sbpostgres flags=(attach_disconnected) {
     /bin/cat Pix -> postgres_shell,
     /usr/bin/admin-mgr Pix -> postgres_shell,
     /nix/store/*/bin/wal-g-2 Pix -> postgres_shell,
+    /nix/store/*/bin/wal-g-3 Pix -> postgres_shell,
     /nix/store/*/bin/pgbackrest Pix -> postgres_shell,
     /nix/store/*/bin/pg_dump ix,
     /usr/bin/pgbackrest Pix -> pgbackrest_shell,
@@ -107,6 +108,7 @@ profile sbpostgres flags=(attach_disconnected) {
         /opt/supabase-admin-agent/supabase-admin-agent-linux-amd64 ix,
         /nix/store/*/bin/.postgres-wrapped ix,
         /nix/store/*/bin/wal-g-2 ix,
+        /nix/store/*/bin/wal-g-3 ix,
         /nix/store/*/bin/pgbackrest ix,
         /nix/store/*/bin/pg_dump ix,
         /nix/store/*/bin/pg_archivecleanup ix,

ansible/tasks/setup-wal-g.yml

@@ -33,16 +33,23 @@
         cmd: sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g-2"
       become: true
 
+    # Installed alongside v2 (INDATA-904). The /usr/local/bin/wal-g symlink below
+    # only points at v3 on orioledb builds; v2 remains the default everywhere else.
+    - name: Install wal-g 3 from nix binary cache
+      ansible.builtin.shell:
+        cmd: sudo -u wal-g bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#wal-g-3"
+      become: true
+
     - name: nix collect garbage
       ansible.builtin.shell:
         cmd: sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix-collect-garbage -d"
       become: true
 
-    - name: Create symlink to make wal-g-v2 the default wal-g
+    - name: Create symlink to make wal-g the default (v3 on orioledb builds, v2 otherwise)
       ansible.builtin.file:
         dest: '/usr/local/bin/wal-g'
         force: true
-        src: '/home/wal-g/.nix-profile/bin/wal-g-2'
+        src: "/home/wal-g/.nix-profile/bin/wal-g-{{ '3' if is_psql_oriole else '2' }}"
         state: 'link'
       become: true
 

nix/checks.nix

@@ -933,6 +933,7 @@
             supabase-cli
             supascan
             wal-g-2
+            wal-g-3
             ;
           devShell = self'.devShells.default;
         }

nix/packages/default.nix

@@ -113,7 +113,7 @@
             inherit (pkgs) yq;
             postgresql_15 = self'.packages."postgresql_15";
           };
-          inherit (pkgs.callPackage ./wal-g.nix { }) wal-g-2;
+          inherit (pkgs.callPackage ./wal-g.nix { }) wal-g-2 wal-g-3;
           inherit (supascan-pkgs) goss supascan supascan-specs;
           inherit (pg-startup-profiler-pkgs) pg-startup-profiler;
           inherit (pkgs.cargo-pgrx)

nix/packages/wal-g.nix

@@ -14,6 +14,10 @@ let
       vendorHash,
       sha256,
       majorVersion,
+      # Extra GOEXPERIMENT flags. wal-g 3.x imports encoding/json/v2 directly
+      # (internal/uploader.go), which Go 1.25 gates behind GOEXPERIMENT=jsonv2.
+      goExperiment ? null,
+      patches ? [ ],
     }:
     buildGoModule rec {
       pname = "wal-g-${majorVersion}";
@@ -26,7 +30,11 @@ let
         inherit sha256;
       };
 
-      inherit vendorHash;
+      inherit vendorHash patches;
+
+      env = lib.optionalAttrs (goExperiment != null) {
+        GOEXPERIMENT = goExperiment;
+      };
 
       nativeBuildInputs = [ installShellFiles ];
 
@@ -77,4 +85,17 @@ in
     vendorHash = "sha256-BbQuY6r30AkxlCZjY8JizaOrqEBdv7rIQet9KQwYB/g=";
     majorVersion = "2";
   };
+
+  # wal-g v3.0.8 — installed alongside v2 (INDATA-904).
+  # OrioleDB support + backward-compat validation; binary is wal-g-3.
+  wal-g-3 = walGCommon {
+    version = "3.0.8";
+    sha256 = "sha256-iDLC3Td/U1msqdpUbJWS+MBDznx7NbddGWFP4rrfSus=";
+    vendorHash = "sha256-K2J/Hi8TQs+UhudgTWsAmPUHKnwKP3cmx21CvDTjs6M=";
+    majorVersion = "3";
+    goExperiment = "jsonv2";
+    # Backport wal-g/wal-g#2112: update OrioleDB incremental-backup page
+    # header parsing to match the beta13 page format.
+    patches = [ ./wal-g/0001-orioledb-update-page-header-format-to-beta13.patch ];
+  };
 }

nix/packages/wal-g/0001-orioledb-update-page-header-format-to-beta13.patch

@@ -0,0 +1,48 @@
+diff --git a/internal/databases/postgres/orioledb/incremental_page_reader.go b/internal/databases/postgres/orioledb/incremental_page_reader.go
+index a53786b760..45ad7e256c 100644
+--- a/internal/databases/postgres/orioledb/incremental_page_reader.go
++++ b/internal/databases/postgres/orioledb/incremental_page_reader.go
+@@ -224,20 +224,7 @@ func (pageReader *incrementalPageReader) WriteDiffMapToHeader(headerWriter io.Wr
+ }
+
+ type pageHeader struct {
+-	state             uint32
+-	usageCount        uint32
+-	pageChangeCount   uint32
+-	checkpointNum     uint32
+-	undoLocation      uint64
+-	csn               uint64
+-	rightLink         uint64
+-	flagsField1Field2 uint32
+-	maxKeyLen         uint16
+-	prevInsertOffset  uint16
+-	chunksCount       uint16
+-	itemsCount        uint16
+-	hikeysEnd         uint16
+-	dataSize          uint16
++	checkpointNum uint32
+ }
+
+ func (header *pageHeader) isValid() bool {
+@@ -249,21 +236,7 @@ func (header *pageHeader) isValid() bool {
+ func parsePageHeader(reader io.Reader) (*pageHeader, error) {
+ 	pageHeader := pageHeader{}
+ 	fields := []parsingutil.FieldToParse{
+-		{Field: &pageHeader.state, Name: "state"},
+-		{Field: &pageHeader.usageCount, Name: "usageCount"},
+-		{Field: &pageHeader.pageChangeCount, Name: "pageChangeCount"},
+-
+ 		{Field: &pageHeader.checkpointNum, Name: "checkpointNum"},
+-		{Field: &pageHeader.undoLocation, Name: "undoLocation"},
+-		{Field: &pageHeader.csn, Name: "csn"},
+-		{Field: &pageHeader.rightLink, Name: "rightLink"},
+-		{Field: &pageHeader.flagsField1Field2, Name: "flagsField1Field2"},
+-		{Field: &pageHeader.maxKeyLen, Name: "maxKeyLen"},
+-		{Field: &pageHeader.prevInsertOffset, Name: "prevInsertOffset"},
+-		{Field: &pageHeader.chunksCount, Name: "chunksCount"},
+-		{Field: &pageHeader.itemsCount, Name: "itemsCount"},
+-		{Field: &pageHeader.hikeysEnd, Name: "hikeysEnd"},
+-		{Field: &pageHeader.dataSize, Name: "dataSize"},
+ 	}
+ 	err := parsingutil.ParseMultipleFieldsFromReader(fields, reader)
+ 	if err != nil {

testinfra/test_ami_nix.py

@@ -1345,19 +1345,21 @@ def test_apparmor_allows_pg_dump(host):
     )
 
 
-def test_apparmor_allows_walg(host):
-    """Verify wal-g-2 can be executed under the sbpostgres AppArmor profile.
+@pytest.mark.parametrize("walg_binary", ["wal-g-2", "wal-g-3"])
+def test_apparmor_allows_walg(host, walg_binary):
+    """Verify wal-g-2 and wal-g-3 can be executed under the sbpostgres AppArmor profile.
 
-    /nix/store/*/bin/wal-g-2 is listed as 'ix' in postgres_shell. We locate the
-    binary at runtime since the Nix store hash is not known ahead of time.
+    /nix/store/*/bin/wal-g-2 and /nix/store/*/bin/wal-g-3 are listed as 'ix' in
+    postgres_shell. We locate the binary at runtime since the Nix store hash is
+    not known ahead of time.
     """
     find_result = run_ssh_command(
         host["ssh"],
-        "find /nix/store -maxdepth 3 -name 'wal-g-2' -type f 2>/dev/null | head -1",
+        f"find /nix/store -maxdepth 3 -name '{walg_binary}' -type f 2>/dev/null | head -1",
     )
     walg_path = find_result["stdout"].strip()
     if not walg_path:
-        print("wal-g-2 not found in Nix store, skipping")
+        print(f"{walg_binary} not found in Nix store, skipping")
         return
 
     result = run_ssh_command(
@@ -1366,7 +1368,7 @@ def test_apparmor_allows_walg(host):
         f"\"COPY (SELECT 1) TO PROGRAM '{walg_path} --version';\"",
     )
     assert result["succeeded"], (
-        f"wal-g-2 was blocked by AppArmor.\n"
+        f"{walg_binary} was blocked by AppArmor.\n"
         f"stdout: {result['stdout']}\nstderr: {result['stderr']}"
     )