feat: test pg_upgrade compatibility with older extension versions

nix/ext/pg_graphql/default.nix

@@ -79,7 +79,7 @@ let
 
         preCheck = ''
           export PGRX_HOME="$(mktemp -d)"
-          export PG_VERSION="${lib.versions.major postgresql.version}"
+          export PG_VERSION="${pgVersion}"
           export NIX_PGLIBDIR="$PGRX_HOME/$PG_VERSION/lib"
           export PATH="$PGRX_HOME/$PG_VERSION/bin:$PATH"
           ${lib.getExe rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ "$PGRX_HOME/$PG_VERSION/"
@@ -119,19 +119,64 @@ let
       }
     );
   allVersions = (builtins.fromJSON (builtins.readFile ../versions.json)).pg_graphql;
+  pgVersion = lib.versions.major postgresql.version;
   supportedVersions = lib.filterAttrs (
-    _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
+    _: value: builtins.elem pgVersion value.postgresql
   ) allVersions;
   versions = lib.naturalSort (lib.attrNames supportedVersions);
   latestVersion = lib.last versions;
   numberOfVersions = builtins.length versions;
   packages = builtins.attrValues (
     lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions
   );
+  buildUnsupported =
+    # Build SQL-only packages for unsupported versions needed by pg_upgrade.
+    # When upgrading PostgreSQL, pg_upgrade requires old extension versions to exist
+    # even if they can't compile against the new PostgreSQL version.
+    version: hash: _rustVersion: _pgrxVersion:
+    stdenv.mkDerivation {
+      inherit pname version;
+      src = fetchFromGitHub {
+        owner = "supabase";
+        repo = pname;
+        rev = "v${version}";
+        inherit hash;
+      };
+      phases = [ "installPhase" ];
+      installPhase = ''
+        mkdir -p $out/share/postgresql/extension
+        for file in $src/sql/*.sql; do
+          filename=$(basename "$file")
+          if [[ "$filename" != "load_sql_config.sql" && "$filename" != "load_sql_context.sql" ]]; then
+            cat "$file"
+            echo ";"
+          fi
+        done > $out/share/postgresql/extension/${pname}--${version}.sql
+      '';
+      meta = with lib; {
+        description = "GraphQL support for PostreSQL";
+        homepage = "https://github.com/supabase/${pname}";
+        license = licenses.postgresql;
+        inherit (postgresql.meta) platforms;
+      };
+    };
+  unsupportedVersions = lib.filterAttrs (
+    _: value: !builtins.elem pgVersion value.postgresql
+  ) allVersions;
+  unsupportedPackages =
+    if pgVersion == 15 then
+      [ ]
+    else
+      # Include SQL-only packages for PG15 extension versions incompatible with current PG
+      builtins.attrValues (
+        lib.mapAttrs (
+          name: value: buildUnsupported name value.hash value.rust value.pgrx
+        ) unsupportedVersions
+      );
 in
 buildEnv {
   name = pname;
-  paths = packages;
+  paths = packages ++ unsupportedPackages;
   pathsToLink = [
     "/lib"
     "/share/postgresql/extension"

nix/ext/pg_jsonschema/default.nix

@@ -49,16 +49,6 @@ let
       # update the following array when the pg_jsonschema version is updated
       # required to ensure that extensions update scripts from previous versions are generated
 
-      previousVersions = [
-        "0.3.1"
-        "0.3.0"
-        "0.2.0"
-        "0.1.4"
-        "0.1.3"
-        "0.1.2"
-        "0.1.1"
-        "0.1.0"
-      ];
       CARGO = "${cargo}/bin/cargo";
       #darwin env needs PGPORT to be unique for build to not clash with other pgrx extensions
       env = lib.optionalAttrs stdenv.isDarwin {
@@ -79,19 +69,15 @@ let
 
       preCheck = ''
         export PGRX_HOME=$(mktemp -d)
-        export NIX_PGLIBDIR=$PGRX_HOME/${lib.versions.major postgresql.version}/lib
-        ${lib.getExe pkgs.rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ $PGRX_HOME/${lib.versions.major postgresql.version}/
-        cargo pgrx init --pg${lib.versions.major postgresql.version} $PGRX_HOME/${lib.versions.major postgresql.version}/bin/pg_config
+        export NIX_PGLIBDIR=$PGRX_HOME/${pgVersion}/lib
+        ${lib.getExe pkgs.rsync} --chmod=ugo+w -a ${postgresql}/ ${postgresql.lib}/ $PGRX_HOME/${pgVersion}/
+        cargo pgrx init --pg${pgVersion} $PGRX_HOME/${pgVersion}/bin/pg_config
       '';
 
       doCheck = true;
 
-      preBuild = ''
-        echo "Processing git tags..."
-        echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt
-      '';
-
       postInstall = ''
+        find $out
         mv $out/lib/${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix}
 
         create_control_files() {
@@ -120,15 +106,18 @@ let
       };
     };
   allVersions = (builtins.fromJSON (builtins.readFile ../versions.json)).pg_jsonschema;
+  pgVersion = lib.versions.major postgresql.version;
   supportedVersions = lib.filterAttrs (
-    _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
+    _: value: builtins.elem pgVersion value.postgresql
   ) allVersions;
   versions = lib.naturalSort (lib.attrNames supportedVersions);
   latestVersion = lib.last versions;
   numberOfVersions = builtins.length versions;
   packages = builtins.attrValues (
     lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions
   );
+  unsupportedVersionsItems = lib.filterAttrs (_: value: value.postgresql == [ "15" ]) allVersions;
+  unsupportedVersions = if pgVersion == "17" then lib.attrNames unsupportedVersionsItems else [ ];
 in
 pkgs.buildEnv {
   name = pname;
@@ -145,6 +134,10 @@ pkgs.buildEnv {
        }"
     )
 
+    for v in ${lib.concatStringsSep " " unsupportedVersions}; do
+      cp $out/share/postgresql/extension/${pname}--${lib.head versions}.sql $out/share/postgresql/extension/${pname}--$v.sql
+    done
+
     create_sql_files() {
       PREVIOUS_VERSION=""
       while IFS= read -r i; do

nix/ext/tests/default.nix

@@ -136,17 +136,16 @@ let
         };
       testScript =
         { nodes, ... }:
-        let
-          pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17";
-        in
         ''
           from pathlib import Path
           versions = {
             "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}],
             "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}],
           }
           extension_name = "${pname}"
-          pg17_configuration = "${pg17-configuration}"
+          system = "${nodes.server.system.build.toplevel}"
+          pg15_configuration = system
+          pg17_configuration = f"{system}/specialisation/postgresql17"
           ext_has_background_worker = ${
             if (installedExtension "15") ? hasBackgroundWorker then "True" else "False"
           }
@@ -198,6 +197,33 @@ let
 
           with subtest("Check pg_regress with postgresql 17 after installing the last version"):
             test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name)
+
+          with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"):
+            # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade
+            for version in versions["15"]:
+              server.systemctl("stop postgresql.service")
+              server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17")
+              server.succeed(
+                f"{pg15_configuration}/bin/switch-to-configuration test >&2"
+              )
+              test.drop_extension()
+              test.install_extension(version)
+              server.succeed(
+                f"{pg17_configuration}/bin/switch-to-configuration test >&2"
+              )
+              has_update_script = server.succeed(
+                "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'"
+              ).strip() == "yes"
+              if has_update_script:
+                # Run the extension update script generated during the upgrade
+                test.run_sql_file("/var/lib/postgresql/update_extensions.sql")
+                # If there was an update script, the last version should be installed
+                test.assert_version_matches(versions["17"][-1])
+              else:
+                # Otherwise, the version should match the version from postgresql 15
+                test.assert_version_matches(version)
+
+              test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name)
         '';
     };
 in
@@ -222,6 +248,7 @@ builtins.listToAttrs (
       "pg_jsonschema"
       "pg_net"
       "pgaudit"
+      "postgis"
       "vector"
       "wrappers"
     ]

nix/ext/tests/lib.py

@@ -47,7 +47,7 @@ def run_sql_file(self, file: str) -> str:
         ).strip()
 
     def drop_extension(self):
-        self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name};")
+        self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name} CASCADE;")
 
     def install_extension(self, version: str):
         self.run_sql(

nix/ext/tests/pgmq.nix

@@ -39,7 +39,7 @@ let
   psql_17 = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17;
 in
 self.inputs.nixpkgs.lib.nixos.runTest {
-  name = "timescaledb";
+  name = "pgmq";
   hostPkgs = pkgs;
   nodes.server =
     { config, ... }:
@@ -105,9 +105,6 @@ self.inputs.nixpkgs.lib.nixos.runTest {
     };
   testScript =
     { nodes, ... }:
-    let
-      pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17";
-    in
     ''
       from pathlib import Path
       versions = {
@@ -116,7 +113,9 @@ self.inputs.nixpkgs.lib.nixos.runTest {
       }
       extension_name = "${pname}"
       support_upgrade = True
-      pg17_configuration = "${pg17-configuration}"
+      system = "${nodes.server.system.build.toplevel}"
+      pg15_configuration = system
+      pg17_configuration = f"{system}/specialisation/postgresql17"
       ext_has_background_worker = ${
         if (installedExtension "15") ? hasBackgroundWorker then "True" else "False"
       }
@@ -173,5 +172,30 @@ self.inputs.nixpkgs.lib.nixos.runTest {
 
       with subtest("Check pg_regress with postgresql 17 after installing the last version"):
         test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name)
+
+      with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"):
+        # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade
+        for version in versions["15"]:
+          server.systemctl("stop postgresql.service")
+          server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17")
+          server.succeed(
+            f"{pg15_configuration}/bin/switch-to-configuration test >&2"
+          )
+          test.drop_extension()
+          test.install_extension(version)
+          server.succeed(
+            f"{pg17_configuration}/bin/switch-to-configuration test >&2"
+          )
+          has_update_script = server.succeed(
+            "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'"
+          ).strip() == "yes"
+          if has_update_script:
+            # Run the extension update script generated during the upgrade
+            test.run_sql_file("/var/lib/postgresql/update_extensions.sql")
+            # If there was an update script, the last version should be installed
+            test.assert_version_matches(versions["17"][-1])
+          else:
+            # Otherwise, the version should match the version from postgresql 15
+            test.assert_version_matches(version)
     '';
 }

nix/ext/tests/pgroonga.nix

@@ -125,17 +125,16 @@ self.inputs.nixpkgs.lib.nixos.runTest {
     };
   testScript =
     { nodes, ... }:
-    let
-      pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17";
-    in
     ''
       from pathlib import Path
       versions = {
         "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}],
         "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}],
       }
       extension_name = "${pname}"
-      pg17_configuration = "${pg17-configuration}"
+      system = "${nodes.server.system.build.toplevel}"
+      pg15_configuration = system
+      pg17_configuration = f"{system}/specialisation/postgresql17"
       ext_has_background_worker = ${
         if (installedExtension "15") ? hasBackgroundWorker then "True" else "False"
       }
@@ -187,5 +186,32 @@ self.inputs.nixpkgs.lib.nixos.runTest {
 
       with subtest("Check pg_regress with postgresql 17 after installing the last version"):
         test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name)
+
+      with subtest("Test pg_upgrade from postgresql 15 to 17 with older extension version"):
+        # Test that all extension versions from postgresql 15 can be upgraded to postgresql 17 using pg_upgrade
+        for version in versions["15"]:
+          server.systemctl("stop postgresql.service")
+          server.succeed("rm -fr /var/lib/postgresql/update_extensions.sql /var/lib/postgresql/17")
+          server.succeed(
+            f"{pg15_configuration}/bin/switch-to-configuration test >&2"
+          )
+          test.drop_extension()
+          test.install_extension(version)
+          server.succeed(
+            f"{pg17_configuration}/bin/switch-to-configuration test >&2"
+          )
+          has_update_script = server.succeed(
+            "test -f /var/lib/postgresql/update_extensions.sql && echo 'yes' || echo 'no'"
+          ).strip() == "yes"
+          if has_update_script:
+            # Run the extension update script generated during the upgrade
+            test.run_sql_file("/var/lib/postgresql/update_extensions.sql")
+            # If there was an update script, the last version should be installed
+            test.assert_version_matches(versions["17"][-1])
+          else:
+            # Otherwise, the version should match the version from postgresql 15
+            test.assert_version_matches(version)
+
+          test.check_pg_regress(Path("${psql_17}/lib/pgxs/src/test/regress/pg_regress"), "17", pg_regress_test_name)
     '';
 }