From c3d444db882422a4dc32451a3671e616a7d21625 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 6 Nov 2025 18:09:20 +0100 Subject: [PATCH 01/10] feat: test pg_upgrade compatibility with older extension versions Add test to verify that all extension versions from PostgreSQL 15 can successfully upgrade to PostgreSQL 17 using pg_upgrade. The test now validates: - Each PG 15 extension version can be upgraded to PG 17 - Extension update scripts are properly generated during upgrade - Version handling works correctly with and without update scripts - Final extension versions match expected values after upgrade --- nix/ext/tests/default.nix | 37 +++++++++-- nix/ext/tests/lib.py | 2 +- nix/ext/tests/pgmq.nix | 34 ++++++++-- nix/ext/tests/pgroonga.nix | 34 ++++++++-- nix/ext/tests/pgsodium.nix | 126 ++++++++++++++++++++++++++----------- nix/ext/tests/vault.nix | 35 +++++++++-- 6 files changed, 213 insertions(+), 55 deletions(-) diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix index 8777a7258..8e06754dc 100644 --- a/nix/ext/tests/default.nix +++ b/nix/ext/tests/default.nix @@ -146,9 +146,6 @@ let }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -157,7 +154,9 @@ let } extension_name = "${pname}" support_upgrade = ${if support_upgrade then "True" else "False"} - pg17_configuration = "${pg17-configuration}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" ext_has_background_worker = ${ if support_upgrade && (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -244,6 +243,33 @@ let with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) '' else "" @@ -271,11 +297,12 @@ builtins.listToAttrs ( "pg_hashids" "pg_jsonschema" "pg_net" + "pg_partman" "pg_repack" "pg_stat_monitor" "pg_tle" "pgaudit" - "pg_partman" + "postgis" "vector" "wal2json" "wrappers" diff --git a/nix/ext/tests/lib.py b/nix/ext/tests/lib.py index 26bd42d49..54db0cee1 100644 --- a/nix/ext/tests/lib.py +++ b/nix/ext/tests/lib.py @@ -55,7 +55,7 @@ def run_sql_file(self, file: str) -> str: ).strip() def drop_extension(self): - self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name};") + self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name} CASCADE;") def install_extension(self, version: str): if self.schema != "public": diff --git a/nix/ext/tests/pgmq.nix b/nix/ext/tests/pgmq.nix index 3e2dc7f91..8af2a70e7 100644 --- a/nix/ext/tests/pgmq.nix +++ b/nix/ext/tests/pgmq.nix @@ -45,7 +45,7 @@ let psql_17 = postgresqlWithExtension self.packages.${pkgs.stdenv.hostPlatform.system}.postgresql_17; in self.inputs.nixpkgs.lib.nixos.runTest { - name = "timescaledb"; + name = "pgmq"; hostPkgs = pkgs; nodes.server = { config, ... }: @@ -111,9 +111,6 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -122,7 +119,9 @@ self.inputs.nixpkgs.lib.nixos.runTest { } extension_name = "${pname}" support_upgrade = True - pg17_configuration = "${pg17-configuration}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -179,5 +178,30 @@ self.inputs.nixpkgs.lib.nixos.runTest { with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) ''; } diff --git a/nix/ext/tests/pgroonga.nix b/nix/ext/tests/pgroonga.nix index 4316d4a72..715d83642 100644 --- a/nix/ext/tests/pgroonga.nix +++ b/nix/ext/tests/pgroonga.nix @@ -131,9 +131,6 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -141,7 +138,9 @@ self.inputs.nixpkgs.lib.nixos.runTest { "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } extension_name = "${pname}" - pg17_configuration = "${pg17-configuration}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -193,5 +192,32 @@ self.inputs.nixpkgs.lib.nixos.runTest { with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; } diff --git a/nix/ext/tests/pgsodium.nix b/nix/ext/tests/pgsodium.nix index 00a2bf44f..d446cd26b 100644 --- a/nix/ext/tests/pgsodium.nix +++ b/nix/ext/tests/pgsodium.nix @@ -47,6 +47,8 @@ let echo 0000000000000000000000000000000000000000000000000000000000000000 '' ); + psql_15 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + psql_17 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; in self.inputs.nixpkgs.lib.nixos.runTest { name = pname; @@ -66,7 +68,21 @@ self.inputs.nixpkgs.lib.nixos.runTest { services.postgresql = { enable = true; - package = postgresqlWithExtension self.packages.${pkgs.stdenv.hostPlatform.system}.postgresql_15; + package = psql_15; + authentication = '' + local all postgres peer map=postgres + local all all peer map=root + ''; + identMap = '' + root root supabase_admin + postgres postgres postgres + ''; + ensureUsers = [ + { + name = "supabase_admin"; + ensureClauses.superuser = true; + } + ]; settings = { "shared_preload_libraries" = pname; "pgsodium.getkey_script" = pgsodiumGetKey; @@ -75,9 +91,7 @@ self.inputs.nixpkgs.lib.nixos.runTest { specialisation.postgresql17.configuration = { services.postgresql = { - package = lib.mkForce ( - postgresqlWithExtension self.packages.${pkgs.stdenv.hostPlatform.system}.postgresql_17 - ); + package = lib.mkForce psql_17; }; systemd.services.postgresql-migrate = { @@ -91,12 +105,8 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; script = let - oldPostgresql = - postgresqlWithExtension - self.packages.${pkgs.stdenv.hostPlatform.system}.postgresql_15; - newPostgresql = - postgresqlWithExtension - self.packages.${pkgs.stdenv.hostPlatform.system}.postgresql_17; + oldPostgresql = psql_15; + newPostgresql = psql_17; oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; in @@ -122,49 +132,93 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' + from pathlib import Path versions = { "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } + extension_name = "${pname}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" + ext_has_background_worker = ${ + if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" + } + sql_test_directory = Path("${../../tests}") + pg_regress_test_name = "${(installedExtension "15").pgRegressTestName or pname}" - def run_sql(query): - return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() - - def check_upgrade_path(pg_version): - with subtest("Check ${pname} upgrade path"): - firstVersion = versions[pg_version][0] - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") - run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" - for version in versions[pg_version][1:]: - run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") - installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") - assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" + ${builtins.readFile ./lib.py} start_all() server.wait_for_unit("multi-user.target") server.wait_for_unit("postgresql.service") - check_upgrade_path("15") + test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory) + + with subtest("Check upgrade path with postgresql 15"): + test.check_upgrade_path("15") - with subtest("Check ${pname} latest extension version"): - server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") - server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") - installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") - latestVersion = versions["15"][-1] - assert f"${pname},{latestVersion}" in installed_extensions + with subtest("Check pg_regress with postgresql 15 after extension upgrade"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) + + last_version = None + with subtest("Check the install of the last version of the extension"): + last_version = test.check_install_last_version("15") + + if ext_has_background_worker: + with subtest("Test switch_${pname}_version"): + test.check_switch_extension_with_background_worker(Path("${psql_15}/lib/${pname}.so"), "15") + + with subtest("Check pg_regress with postgresql 15 after installing the last version"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) with subtest("switch to postgresql 17"): server.succeed( - "${pg17-configuration}/bin/switch-to-configuration test >&2" + f"{pg17_configuration}/bin/switch-to-configuration test >&2" ) - check_upgrade_path("17") + with subtest("Check last version of the extension after postgresql upgrade"): + test.assert_version_matches(last_version) + + with subtest("Check upgrade path with postgresql 17"): + test.check_upgrade_path("17") + + with subtest("Check pg_regress with postgresql 17 after extension upgrade"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Check the install of the last version of the extension"): + test.check_install_last_version("17") + + with subtest("Check pg_regress with postgresql 17 after installing the last version"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; } diff --git a/nix/ext/tests/vault.nix b/nix/ext/tests/vault.nix index 42fafacab..fa519d12a 100644 --- a/nix/ext/tests/vault.nix +++ b/nix/ext/tests/vault.nix @@ -139,9 +139,6 @@ self.inputs.nixpkgs.lib.nixos.runTest { }; testScript = { nodes, ... }: - let - pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; - in '' from pathlib import Path versions = { @@ -149,8 +146,10 @@ self.inputs.nixpkgs.lib.nixos.runTest { "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], } extension_name = "${pname}" + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" support_upgrade = True - pg17_configuration = "${pg17-configuration}" ext_has_background_worker = ${ if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" } @@ -203,5 +202,33 @@ self.inputs.nixpkgs.lib.nixos.runTest { with subtest("Check pg_regress with postgresql 17 after installing the last version"): test.run_sql_file("${../../../ansible/files/postgresql_extension_custom_scripts/supabase_vault/after-create.sql}") test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + + test.run_sql_file("${../../../ansible/files/postgresql_extension_custom_scripts/supabase_vault/after-create.sql}") + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; } From 08457dbbbcd71cbba5e8ac55d1a125023aeaa613 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 6 Nov 2025 19:17:02 +0100 Subject: [PATCH 02/10] fix(pg_graphql): include unsupported versions as SQL-only for pg_upgrade compatibility When upgrading PostgreSQL versions, pg_upgrade needs access to old extension versions sql to migrate from. This adds unsupported pg_graphql versions (those not compilable with current PostgreSQL) as SQL-only packages, ensuring migration paths exist when upgrading from older PostgreSQL versions. --- nix/ext/pg_graphql/default.nix | 51 ++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/nix/ext/pg_graphql/default.nix b/nix/ext/pg_graphql/default.nix index a7f6d1065..898765d93 100644 --- a/nix/ext/pg_graphql/default.nix +++ b/nix/ext/pg_graphql/default.nix @@ -79,7 +79,7 @@ let preCheck = '' export PGRX_HOME="$(mktemp -d)" - export PG_VERSION="${lib.versions.major postgresql.version}" + export PG_VERSION="${pgVersion}" export NIX_PGLIBDIR="$PGRX_HOME/$PG_VERSION/lib" export PATH="$PGRX_HOME/$PG_VERSION/bin:$PATH" ${lib.getExe rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ "$PGRX_HOME/$PG_VERSION/" @@ -119,8 +119,9 @@ let } ); allVersions = (builtins.fromJSON (builtins.readFile ../versions.json)).pg_graphql; + pgVersion = lib.versions.major postgresql.version; supportedVersions = lib.filterAttrs ( - _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + _: value: builtins.elem pgVersion value.postgresql ) allVersions; versions = lib.naturalSort (lib.attrNames supportedVersions); latestVersion = lib.last versions; @@ -128,10 +129,54 @@ let packages = builtins.attrValues ( lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); + buildUnsupported = + # Build SQL-only packages for unsupported versions needed by pg_upgrade. + # When upgrading PostgreSQL, pg_upgrade requires old extension versions to exist + # even if they can't compile against the new PostgreSQL version. + version: hash: _rustVersion: _pgrxVersion: + stdenv.mkDerivation { + inherit pname version; + src = fetchFromGitHub { + owner = "supabase"; + repo = pname; + rev = "v${version}"; + inherit hash; + }; + phases = [ "installPhase" ]; + installPhase = '' + mkdir -p $out/share/postgresql/extension + for file in $src/sql/*.sql; do + filename=$(basename "$file") + if [[ "$filename" != "load_sql_config.sql" && "$filename" != "load_sql_context.sql" ]]; then + cat "$file" + echo ";" + fi + done > $out/share/postgresql/extension/${pname}--${version}.sql + ''; + meta = with lib; { + description = "GraphQL support for PostreSQL"; + homepage = "https://github.com/supabase/${pname}"; + license = licenses.postgresql; + inherit (postgresql.meta) platforms; + }; + }; + unsupportedVersions = lib.filterAttrs ( + _: value: !builtins.elem pgVersion value.postgresql + ) allVersions; + unsupportedPackages = + if pgVersion == 15 then + [ ] + else + # Include SQL-only packages for PG15 extension versions incompatible with current PG + builtins.attrValues ( + lib.mapAttrs ( + name: value: buildUnsupported name value.hash value.rust value.pgrx + ) unsupportedVersions + ); in (buildEnv { name = pname; - paths = packages; + paths = packages ++ unsupportedPackages; pathsToLink = [ "/lib" "/share/postgresql/extension" From dbd3d51f2a18023a8feda2b6471e022c8329ac3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 6 Nov 2025 23:03:52 +0100 Subject: [PATCH 03/10] fix(pg_jsonschema): include unsupported versions as SQL-only for pg_upgrade compatibility When upgrading PostgreSQL versions, pg_upgrade needs access to old extension versions sql to migrate from. This adds unsupported pg_graphql versions (those not compilable with current PostgreSQL) as SQL-only packages, ensuring migration paths exist when upgrading from older PostgreSQL versions. --- nix/ext/pg_jsonschema/default.nix | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/nix/ext/pg_jsonschema/default.nix b/nix/ext/pg_jsonschema/default.nix index d3a72036f..38e2f923c 100644 --- a/nix/ext/pg_jsonschema/default.nix +++ b/nix/ext/pg_jsonschema/default.nix @@ -75,9 +75,9 @@ let preCheck = '' export PGRX_HOME=$(mktemp -d) - export NIX_PGLIBDIR=$PGRX_HOME/${lib.versions.major postgresql.version}/lib - ${lib.getExe pkgs.rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ $PGRX_HOME/${lib.versions.major postgresql.version}/ - cargo pgrx init --pg${lib.versions.major postgresql.version} $PGRX_HOME/${lib.versions.major postgresql.version}/bin/pg_config + export NIX_PGLIBDIR=$PGRX_HOME/${pgVersion}/lib + ${lib.getExe pkgs.rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ $PGRX_HOME/${pgVersion}/ + cargo pgrx init --pg${pgVersion} $PGRX_HOME/${pgVersion}/bin/pg_config ''; # Tests are disabled for specific versions because pgrx tests require @@ -92,12 +92,8 @@ let "0.3.3" ]); - preBuild = '' - echo "Processing git tags..." - echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt - ''; - postInstall = '' + find $out mv $out/lib/${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix} create_control_files() { @@ -126,8 +122,9 @@ let }; }; allVersions = (builtins.fromJSON (builtins.readFile ../versions.json)).pg_jsonschema; + pgVersion = lib.versions.major postgresql.version; supportedVersions = lib.filterAttrs ( - _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + _: value: builtins.elem pgVersion value.postgresql ) allVersions; versions = lib.naturalSort (lib.attrNames supportedVersions); latestVersion = lib.last versions; @@ -135,6 +132,8 @@ let packages = builtins.attrValues ( lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); + unsupportedVersionsItems = lib.filterAttrs (_: value: value.postgresql == [ "15" ]) allVersions; + unsupportedVersions = if pgVersion == "17" then lib.attrNames unsupportedVersionsItems else [ ]; in (pkgs.buildEnv { name = pname; @@ -151,6 +150,10 @@ in }" ) + for v in ${lib.concatStringsSep " " unsupportedVersions}; do + cp $out/share/postgresql/extension/${pname}--${lib.head versions}.sql $out/share/postgresql/extension/${pname}--$v.sql + done + create_sql_files() { PREVIOUS_VERSION="" while IFS= read -r i; do From 2f96685d9c0250215a7dd358351e3cc4fe301a99 Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Fri, 21 Nov 2025 15:41:19 +0100 Subject: [PATCH 04/10] fix(pg_stat_monitor): exclude v1.0 from pg_upgrade compatibility tests When upgrading from PostgreSQL 15 to 17, pg_stat_monitor version 1.0 (PG 15-only) cannot be migrated as it uses .sql.in template files that reference MODULE_PATHNAME without proper processing for the target version. This marks version 1.0 as not pg_upgrade compatible and filters it from the version test list, allowing the test to use version 2.1 (which supports both PG 15 and 17) for pg_upgrade validation instead. Version 1.0 remains available for PG 15 installations. Version 2.1 has different schemas on PostgreSQL 15 vs 17 despite sharing the same version number. On PG 15 it uses the older schema with blk_read_time and blk_write_time columns, while on PG 17 it uses a newer schema with shared_blk_read_time, shared_blk_write_time, local_blk_read_time, local_blk_write_time and additional JIT statistics columns. During pg_upgrade from PG 15 to 17, the extension version remains 2.1 without schema migration since no update script is generated. Fresh installations on PG 17 receive the new schema while pg_upgrade retains the old schema, creating a test conflict as both scenarios share the same expected output file. A custom test implementation skips pg_regress validation after pg_upgrade when no update script is generated, since the schema mismatch is expected behavior. This maintains full test coverage for fresh installations through the regular psql_17 check while validating extension version compatibility for pg_upgrade scenarios. --- nix/ext/pg_stat_monitor.nix | 11 +- nix/ext/tests/pg_stat_monitor.nix | 265 ++++++++++++++++++++++++++++++ nix/ext/versions.json | 3 +- 3 files changed, 275 insertions(+), 4 deletions(-) create mode 100644 nix/ext/tests/pg_stat_monitor.nix diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix index ddf46de30..eaedcb991 100644 --- a/nix/ext/pg_stat_monitor.nix +++ b/nix/ext/pg_stat_monitor.nix @@ -17,11 +17,16 @@ let ) allVersions; # Derived version information - versions = lib.naturalSort (lib.attrNames supportedVersions); - latestVersion = lib.last versions; + allVersionsList = lib.naturalSort (lib.attrNames supportedVersions); + versions = builtins.filter (v: (allVersions.${v}.pgUpgradeCompatible or true)) allVersionsList; + latestVersion = lib.last allVersionsList; numberOfVersions = builtins.length versions; + # Filter to only build pg_upgrade compatible versions + pgUpgradeCompatibleVersions = lib.filterAttrs ( + name: _: allVersions.${name}.pgUpgradeCompatible or true + ) supportedVersions; packages = builtins.attrValues ( - lib.mapAttrs (name: value: build name value.hash value.revision) supportedVersions + lib.mapAttrs (name: value: build name value.hash value.revision) pgUpgradeCompatibleVersions ); # Build function for individual versions diff --git a/nix/ext/tests/pg_stat_monitor.nix b/nix/ext/tests/pg_stat_monitor.nix new file mode 100644 index 000000000..27b2003f6 --- /dev/null +++ b/nix/ext/tests/pg_stat_monitor.nix @@ -0,0 +1,265 @@ +{ self, pkgs }: +# +# Custom test for pg_stat_monitor extension +# +# IMPORTANT: This extension requires special handling because version 2.1 has different +# schemas on PostgreSQL 15 vs 17, even though they share the same version number: +# +# - PG 15 (version 2.1): Uses older schema with `blk_read_time`, `blk_write_time` +# - PG 17 (version 2.1): Uses newer schema with `shared_blk_read_time`, +# `shared_blk_write_time`, `local_blk_read_time`, `local_blk_write_time`, plus +# additional JIT columns (`jit_deform_count`, `jit_deform_time`, `stats_since`, +# `minmax_stats_since`) +# +# During pg_upgrade from PG 15 to PG 17: +# - The extension remains at version 2.1 +# - No update script is generated (same version number) +# - The schema stays as PG 15 (old schema) +# - Fresh installs on PG 17 get the new schema +# +# This creates a conflict: the same expected output file (`z_17_pg_stat_monitor.out`) +# is used for both scenarios, but they produce different schemas. +# +# Solution: Skip pg_regress after pg_upgrade when no update script is generated, +# since the schema won't match the expected output for fresh PG 17 installs. +# This still validates that: +# - The extension version is correct after upgrade +# - Fresh installs work correctly (tested by regular psql_17 check) +# - Upgrade paths work correctly +# +let + pname = "pg_stat_monitor"; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: + self.legacyPackages.${pkgs.system}."psql_${postgresMajorVersion}".exts."${pname}"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + ]; + passthru = { + inherit (postgresql) version psqlSchema; + installedExtensions = [ (installedExtension majorVersion) ]; + lib = pkg; + withPackages = _: pkg; + withJIT = pkg; + withoutJIT = pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; + psql_15 = postgresqlWithExtension self.packages.${pkgs.stdenv.hostPlatform.system}.postgresql_15; + psql_17 = postgresqlWithExtension self.packages.${pkgs.stdenv.hostPlatform.system}.postgresql_17; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + services.openssh = { + enable = true; + }; + + services.postgresql = { + enable = true; + package = psql_15; + enableTCPIP = true; + authentication = '' + local all postgres peer map=postgres + local all all peer map=root + ''; + identMap = '' + root root supabase_admin + postgres postgres postgres + ''; + ensureUsers = [ + { + name = "supabase_admin"; + ensureClauses.superuser = true; + } + ]; + settings = (installedExtension "15").defaultSettings or { }; + }; + + networking.firewall.allowedTCPPorts = [ config.services.postgresql.settings.port ]; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce psql_17; + settings = (installedExtension "17").defaultSettings or { }; + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = psql_15; + newPostgresql = psql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" \ + ${ + if config.services.postgresql.settings.shared_preload_libraries != null then + " --old-options='-c shared_preload_libraries=${config.services.postgresql.settings.shared_preload_libraries}' --new-options='-c shared_preload_libraries=${config.services.postgresql.settings.shared_preload_libraries}'" + else + "" + } + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + }; + testScript = + { nodes, ... }: + '' + from pathlib import Path + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + } + extension_name = "${pname}" + support_upgrade = True + system = "${nodes.server.system.build.toplevel}" + pg15_configuration = system + pg17_configuration = f"{system}/specialisation/postgresql17" + ext_has_background_worker = ${ + if (installedExtension "15") ? hasBackgroundWorker then "True" else "False" + } + sql_test_directory = Path("${../../tests}") + pg_regress_test_name = "${(installedExtension "15").pgRegressTestName or pname}" + ext_schema = "${(installedExtension "15").defaultSchema or "public"}" + lib_name = "${(installedExtension "15").libName or pname}" + print(f"Running tests for extension: {lib_name}") + + ${builtins.readFile ./lib.py} + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade, ext_schema) + test.create_schema() + + with subtest("Check upgrade path with postgresql 15"): + test.check_upgrade_path("15") + + with subtest("Check pg_regress with postgresql 15 after extension upgrade"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) + + last_version = None + with subtest("Check the install of the last version of the extension"): + last_version = test.check_install_last_version("15") + + if ext_has_background_worker: + with subtest("Test switch_${pname}_version"): + test.check_switch_extension_with_background_worker(Path(f"${psql_15}/lib/{lib_name}.so"), "15") + + with subtest("Check pg_regress with postgresql 15 after installing the last version"): + test.check_pg_regress(Path("${psql_15}/lib/pgxs/src/test/regress/pg_regress"), "15", pg_regress_test_name) + + with subtest("switch to postgresql 17"): + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + + with subtest("Check last version of the extension after postgresql upgrade"): + test.assert_version_matches(last_version) + + with subtest("Check upgrade path with postgresql 17"): + test.check_upgrade_path("17") + + with subtest("Check pg_regress with postgresql 17 after extension upgrade"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Check the install of the last version of the extension"): + test.check_install_last_version("17") + + with subtest("Check pg_regress with postgresql 17 after installing the last version"): + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + + with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"): + # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade + for version in versions["15"]: + server.systemctl("stop postgresql.service") + server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17") + server.succeed( + f"{pg15_configuration}/bin/switch-to-configuration test >&2" + ) + test.drop_extension() + test.install_extension(version) + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + has_update_script = server.succeed( + "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'" + ).strip() == "yes" + if has_update_script: + # Run the extension update script generated during the upgrade + test.run_sql_file("/var/lib/postgresql/update_extensions.sql") + # If there was an update script, the last version should be installed + test.assert_version_matches(versions["17"][-1]) + # With update script, the schema should match PG 17 expectations + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) + else: + # Otherwise, the version should match the version from postgresql 15 + test.assert_version_matches(version) + # Skip pg_regress when no update script is generated because: + # - The extension retains the PG 15 schema (old column names) + # - The expected output file expects PG 17 schema (new column names) + # - This mismatch is expected behavior - pg_upgrade doesn't change schemas + # when version numbers don't change + # - The extension is still functional, just with the older schema + print(f"Skipping pg_regress for {extension_name} after pg_upgrade without update script") + print(f"Version {version} retains PG 15 schema, which differs from PG 17 fresh install schema") + ''; +} diff --git a/nix/ext/versions.json b/nix/ext/versions.json index b35c6c40b..e83366a0c 100644 --- a/nix/ext/versions.json +++ b/nix/ext/versions.json @@ -775,7 +775,8 @@ "15" ], "revision": "1.0.1", - "hash": "sha256-sQEpIknAFOmvNTX2G23X4BvMdy3Ms7sXx7hLZt8jyUk=" + "hash": "sha256-sQEpIknAFOmvNTX2G23X4BvMdy3Ms7sXx7hLZt8jyUk=", + "pgUpgradeCompatible": false }, "2.1": { "postgresql": [ From d29409d6637fd728f04e7ea0b2880b6a2d7f5785 Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Mon, 19 Jan 2026 11:34:58 +0100 Subject: [PATCH 05/10] fix(pg_stat_monitor): address review comments for latestVersion and lib_name parameters --- nix/ext/pg_stat_monitor.nix | 2 +- nix/ext/tests/pg_stat_monitor.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix index eaedcb991..dc6ba8b1e 100644 --- a/nix/ext/pg_stat_monitor.nix +++ b/nix/ext/pg_stat_monitor.nix @@ -19,7 +19,7 @@ let # Derived version information allVersionsList = lib.naturalSort (lib.attrNames supportedVersions); versions = builtins.filter (v: (allVersions.${v}.pgUpgradeCompatible or true)) allVersionsList; - latestVersion = lib.last allVersionsList; + latestVersion = lib.last versions; numberOfVersions = builtins.length versions; # Filter to only build pg_upgrade compatible versions pgUpgradeCompatibleVersions = lib.filterAttrs ( diff --git a/nix/ext/tests/pg_stat_monitor.nix b/nix/ext/tests/pg_stat_monitor.nix index 27b2003f6..1bb8a57ee 100644 --- a/nix/ext/tests/pg_stat_monitor.nix +++ b/nix/ext/tests/pg_stat_monitor.nix @@ -187,7 +187,7 @@ self.inputs.nixpkgs.lib.nixos.runTest { server.wait_for_unit("multi-user.target") server.wait_for_unit("postgresql.service") - test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade, ext_schema) + test = PostgresExtensionTest(server, extension_name, versions, sql_test_directory, support_upgrade, ext_schema, lib_name) test.create_schema() with subtest("Check upgrade path with postgresql 15"): From e1e465ba969fd3382686765754a8ab3284aab46a Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Mon, 19 Jan 2026 12:24:10 +0100 Subject: [PATCH 06/10] fix(pg_jsonschema): broaden unsupported version detection beyond PG15-only versions --- nix/ext/pg_jsonschema/default.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nix/ext/pg_jsonschema/default.nix b/nix/ext/pg_jsonschema/default.nix index 38e2f923c..4ea675b9a 100644 --- a/nix/ext/pg_jsonschema/default.nix +++ b/nix/ext/pg_jsonschema/default.nix @@ -132,8 +132,10 @@ let packages = builtins.attrValues ( lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions ); - unsupportedVersionsItems = lib.filterAttrs (_: value: value.postgresql == [ "15" ]) allVersions; - unsupportedVersions = if pgVersion == "17" then lib.attrNames unsupportedVersionsItems else [ ]; + unsupportedVersionsItems = lib.filterAttrs ( + _: value: !(builtins.elem pgVersion value.postgresql) + ) allVersions; + unsupportedVersions = lib.attrNames unsupportedVersionsItems; in (pkgs.buildEnv { name = pname; From 39b92b21c71eda4e6e84563e5946b4efa6a5fbf6 Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Mon, 19 Jan 2026 13:10:06 +0100 Subject: [PATCH 07/10] fix(pg_jsonschema): remove debug find command from build output --- nix/ext/pg_jsonschema/default.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/ext/pg_jsonschema/default.nix b/nix/ext/pg_jsonschema/default.nix index 4ea675b9a..d3b272645 100644 --- a/nix/ext/pg_jsonschema/default.nix +++ b/nix/ext/pg_jsonschema/default.nix @@ -93,7 +93,6 @@ let ]); postInstall = '' - find $out mv $out/lib/${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix} create_control_files() { From dc1c6d0ddc9a5c3193cb9e44fc73875c9b44162f Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Wed, 28 Jan 2026 15:36:43 +0100 Subject: [PATCH 08/10] fix(pg_stat_monitor): use custom test instead of default for pg_upgrade compatibility Remove pg_stat_monitor from default extension test list to prevent conflict with the custom pg_stat_monitor.nix test file. The custom test includes special logic to skip pg_regress after pg_upgrade when no update script is generated, which is necessary because version 2.1 has different schemas between PostgreSQL 15 and 17 despite sharing the same version number. --- nix/ext/tests/default.nix | 1 - 1 file changed, 1 deletion(-) diff --git a/nix/ext/tests/default.nix b/nix/ext/tests/default.nix index 8e06754dc..16b8b3bc0 100644 --- a/nix/ext/tests/default.nix +++ b/nix/ext/tests/default.nix @@ -299,7 +299,6 @@ builtins.listToAttrs ( "pg_net" "pg_partman" "pg_repack" - "pg_stat_monitor" "pg_tle" "pgaudit" "postgis" From 2fc828bd94ac8c5a763591913355ad92f550ff6b Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Wed, 28 Jan 2026 16:37:51 +0100 Subject: [PATCH 09/10] fix(pgmq): run after-create script before pg_regress test The test was failing because the pg_regress test expects the pgmq extension to be in a fully functional state, but after pg_upgrade the after-create script wasn't being run. This script sets up necessary pgmq configurations that the tests depend on. --- nix/ext/tests/pgmq.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nix/ext/tests/pgmq.nix b/nix/ext/tests/pgmq.nix index 8af2a70e7..1ba83f153 100644 --- a/nix/ext/tests/pgmq.nix +++ b/nix/ext/tests/pgmq.nix @@ -203,5 +203,8 @@ self.inputs.nixpkgs.lib.nixos.runTest { else: # Otherwise, the version should match the version from postgresql 15 test.assert_version_matches(version) + + test.run_sql_file("${../../../ansible/files/postgresql_extension_custom_scripts/pgmq/after-create.sql}") + test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name) ''; } From 66d2f75c6ccadc819333c23c518b0fa1b3c81cc5 Mon Sep 17 00:00:00 2001 From: Yvan Sraka Date: Wed, 28 Jan 2026 16:37:58 +0100 Subject: [PATCH 10/10] fix(pg_graphql): use string comparison for pgVersion pgVersion is a string, so comparison should be string-to-string instead of string-to-integer. --- nix/ext/pg_graphql/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/ext/pg_graphql/default.nix b/nix/ext/pg_graphql/default.nix index 898765d93..3e0ee0329 100644 --- a/nix/ext/pg_graphql/default.nix +++ b/nix/ext/pg_graphql/default.nix @@ -164,7 +164,7 @@ let _: value: !builtins.elem pgVersion value.postgresql ) allVersions; unsupportedPackages = - if pgVersion == 15 then + if pgVersion == "15" then [ ] else # Include SQL-only packages for PG15 extension versions incompatible with current PG