Upgrade to PostgreSQL 16 (#313)

cloud/aws/bin/deploy.py

@@ -1,22 +1,20 @@
-import sys
+import textwrap
+import os
 
 from cloud.aws.templates.aws_oidc.bin import resources
 from cloud.aws.templates.aws_oidc.bin.aws_cli import AwsCli
 from cloud.shared.bin.lib import terraform
 from cloud.shared.bin.lib.print import print
 from cloud.shared.bin.lib.color import Color
+from cloud.shared.bin.lib.config_loader import ConfigLoader
 
 
-def run(config):
+def run(config: ConfigLoader):
     aws = AwsCli(config)
 
     if not config.is_test():
-        secret_length = aws.get_application_secret_length()
-        if secret_length < 32:
-            print(
-                f'{Color.RED}The application secret must be at least 32 characters in length, and ideally 64 characters. The current secret has a length of {secret_length}. See https://docs.civiform.us/it-manual/sre-playbook/initial-deployment/terraform-deploy-system#rotating-the-application-secret for details on how to regenerate the secret with a longer length.{Color.END}'
-            )
-            exit(1)
+        _check_application_secret_length(config, aws)
+        _check_for_postgres_upgrade(config, aws)
 
     if not terraform.perform_apply(config):
         print('Terraform deployment failed.')
@@ -33,3 +31,72 @@ def run(config):
     print(
         f'Server is available at {lb_dns}. Check your domain registrar to ensure your CNAME record for {base_url} points to this address.'
     )
+
+
+def _check_application_secret_length(config: ConfigLoader, aws: AwsCli):
+    if not config.is_test():
+        secret_length = aws.get_application_secret_length()
+        if secret_length < 32:
+            print(
+                f'{Color.RED}The application secret must be at least 32 characters in length, and ideally 64 characters. The current secret has a length of {secret_length}. See https://docs.civiform.us/it-manual/sre-playbook/initial-deployment/terraform-deploy-system#rotating-the-application-secret for details on how to regenerate the secret with a longer length.{Color.END}'
+            )
+            exit(1)
+
+
+def _check_for_postgres_upgrade(config: ConfigLoader, aws: AwsCli):
+    # https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_UpgradeDBInstance.PostgreSQL.html
+    # For each major PG version, the oldest allowed PG minor you must be on in order to upgrade to that major version.
+    # We don't really care which minor version of the upgraded version you get upgraded to, as AWS will take care of upgrading
+    # the minor version later (e.g. 12.17 can only upgrade to 16.1, but then AWS will upgrade that again to 16.2
+    # automatically once it's at 16.1 in the next maintenance window).
+    #
+    # We are currently only upgrading 12 -> 16. Fill in this table as needed for future upgrades.
+    #
+    # major_version_to_upgrade_to: {current_major_version: oldest_allowable_minor_version}
+    pg_upgrade_table = {16: {12: 17}}
+    postgresql_major_to_apply = config.get_config_var(
+        "POSTGRESQL_MAJOR_VERSION") or terraform.find_variable_default(
+            config, 'postgresql_major_version')
+    if postgresql_major_to_apply:
+        to_apply = int(postgresql_major_to_apply)
+        current_major, current_minor = aws.get_postgresql_version(
+            f'{config.app_prefix}-{resources.DATABASE}')
+        if to_apply != current_major:
+            print(
+                textwrap.dedent(
+                    f'''
+                {Color.CYAN}This version of CiviForm contains an upgrade to PostgreSQL {to_apply}. Your install is currently using PostgreSQL version {current_major}.{current_minor}.
+                
+                The upgrade may take an extra 10-20 minutes to complete, during which time the CiviForm application will be unavailable. Before upgrading, ensure you have a backup of your database. You can do this by running bin/run and choosing the dumpdb command.
+                Additionally, a snapshot will be performed just prior to the upgrade. The snapshot will have a name that starts with "preupgrade". You may also have a snapshot called "{config.app_prefix}-civiform-db-finalsnapshot".
+                {Color.END}
+                '''))
+            if to_apply < current_major:
+                raise ValueError(
+                    f'{Color.RED}Your current version of PostgreSQL appears to be newer than the version specified for this CiviForm release. Ensure you are using the correct version of the cloud-deploy-infra repo and POSTGRESQL_MAJOR_VERSION is unset or set appropriately.{Color.END}'
+                )
+            if to_apply not in pg_upgrade_table:
+                raise ValueError(
+                    f'{Color.RED}Unsupported upgrade to PostgreSQL version {to_apply} specified for POSTGRESQL_MAJOR_VERSION. If this seems incorrect, contact a CiviForm maintainer.{Color.END}'
+                )
+            if current_major not in pg_upgrade_table[to_apply]:
+                answer = input(
+                    f'{Color.YELLOW}This version of the deployment tool does not have information about if {current_major}.{current_minor} is sufficiently new enough to upgrade to version {to_apply}. Check https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_UpgradeDBInstance.PostgreSQL.html and verify this is a valid upgrade path. Would you like to proceed with the upgrade? (y/N): {Color.END}'
+                )
+                if answer.lower() != 'y':
+                    exit(1)
+            elif current_minor < pg_upgrade_table[to_apply][current_major]:
+                print(
+                    f'{Color.RED}In order to upgrade to version {to_apply}, you must first upgrade to at least PostgreSQL {current_major}.{pg_upgrade_table[to_apply][current_major]}. You will need to perform this upgrade in the AWS RDS console before proceeding.{Color.END}'
+                )
+                exit(1)
+            # If a user sets ALLOW_POSTGRESQL_UPGRADE in their config file, config.get_config_var will pick it up.
+            # If they've set it as an environment variable, we need to detect that and then add it to the config
+            # object ourselves so that it is picked up with the manifest is compiled.
+            if config.get_config_var("ALLOW_POSTGRESQL_UPGRADE") != "true":
+                answer = input(
+                    f'{Color.YELLOW}Would you like to proceed with the upgrade? (y/N): {Color.END}'
+                )
+                if answer.lower() not in ['y', 'yes']:
+                    exit(2)
+                config.add_config_value("ALLOW_POSTGRESQL_UPGRADE", "true")

cloud/aws/templates/aws_oidc/bin/aws_cli.py

@@ -325,6 +325,20 @@ def delete_table(self, table_name: str) -> bool:
                 print(f'Error deleting DynamoDB table: {e.stdout.decode()}')
                 return False
 
+    def get_postgresql_version(self, db_name: str) -> str:
+        try:
+            res = self._call_cli(
+                f"rds describe-db-instances --db-instance-identifier={db_name} --query 'DBInstances[0].{{EngineVersion:EngineVersion}}'"
+            )
+            ver_str = res["EngineVersion"]
+            maj, min = re.match(r'^(\d+)\.?(\d+)?', ver_str).groups()
+            maj = int(maj)
+            min = int(min) if min else 0
+            return (maj, min)
+        except subprocess.CalledProcessError as e:
+            print(f'Error getting Postgres version: {e.stdout.decode()}')
+            return -1
+
     def get_dbaccess_ec2_host_ip(self) -> str:
         return self._call_cli(
             "ec2 describe-instances --filters 'Name=tag:Module,Values=dbaccess' 'Name=instance-state-name,Values=running' --query 'Reservations[0].Instances[0].PublicIpAddress'"

cloud/aws/templates/aws_oidc/main.tf

@@ -9,18 +9,22 @@ locals {
 # List of params that we could configure:
 # https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Appendix.PostgreSQL.CommonDBATasks.Parameters.html#Appendix.PostgreSQL.CommonDBATasks.Parameters.parameters-list
 resource "aws_db_parameter_group" "civiform" {
-  name = "${var.app_prefix}-civiform-db-params"
+  name_prefix = "${var.app_prefix}-civiform-db-params"
   tags = {
     Name = "${var.app_prefix} Civiform DB Parameters"
     Type = "Civiform DB Parameters"
   }
 
-  family = "postgres12"
+  family = "postgres${var.postgresql_major_version}"
 
   parameter {
     name  = "log_connections"
     value = "1"
   }
+
+  lifecycle {
+    create_before_destroy = true
+  }
 }
 
 resource "aws_db_instance" "civiform" {
@@ -30,6 +34,8 @@ resource "aws_db_instance" "civiform" {
     Type = "Civiform Database"
   }
 
+  apply_immediately = true
+
   # If not null, destroys the current database, replacing it with a new one restored from the provided snapshot
   snapshot_identifier             = var.postgres_restore_snapshot_identifier
   deletion_protection             = local.deletion_protection
@@ -40,7 +46,8 @@ resource "aws_db_instance" "civiform" {
   storage_throughput              = var.aws_db_storage_throughput
   iops                            = var.aws_db_iops
   engine                          = "postgres"
-  engine_version                  = "12"
+  engine_version                  = var.postgresql_major_version
+  allow_major_version_upgrade     = var.allow_postgresql_upgrade
   username                        = aws_secretsmanager_secret_version.postgres_username_secret_version.secret_string
   password                        = aws_secretsmanager_secret_version.postgres_password_secret_version.secret_string
   vpc_security_group_ids          = [aws_security_group.rds.id]

cloud/aws/templates/aws_oidc/variable_definitions.json

@@ -386,5 +386,17 @@
     "secret": false,
     "tfvar": true,
     "type": "integer"
+  },
+  "ALLOW_POSTGRESQL_UPGRADE": {
+    "required": false,
+    "secret": false,
+    "tfvar": true,
+    "type": "bool"
+  },
+  "POSTGRESQL_MAJOR_VERSION": {
+    "required": false,
+    "secret": false,
+    "tfvar": true,
+    "type": "integer"
   }
 }

cloud/aws/templates/aws_oidc/variables.tf

@@ -494,3 +494,16 @@ variable "dbaccess" {
   description = "Whether to set up resources to allow access to the database from an EC2 host"
   default     = false
 }
+
+
+variable "allow_postgresql_upgrade" {
+  type        = bool
+  description = "Allow major version upgrade for PostgreSQL"
+  default     = false
+}
+
+variable "postgresql_major_version" {
+  type        = number
+  description = "Major version of PostgreSQL to use"
+  default     = 16
+}

cloud/shared/bin/lib/config_loader.py

@@ -12,6 +12,7 @@
 
 from cloud.shared.bin.lib.config_parser import ConfigParser
 from cloud.shared.bin.lib.print import print
+from cloud.shared.bin.lib.write_tfvars import TfVarWriter
 from cloud.shared.bin.lib.variable_definition_loader import \
     load_variables_definitions
 
@@ -72,6 +73,11 @@ def _load_config_fields(self, config_file: str):
         self._export_env_variables(config_fields)
         return config_fields
 
+    def add_config_value(self, key: str, value: str):
+        self._config_fields[key] = value
+        os.environ[key] = value
+        self.write_tfvars_file()
+
     # TODO(https://github.com/civiform/civiform/issues/4293): remove this when
     # the local deploy system does not read values from env variables anymore.
     # Currently some env variables are read from local deploy code (legacy
@@ -396,3 +402,9 @@ def get_template_dir(self):
         if template_dir is None or not os.path.exists(template_dir):
             exit(f"Could not find template directory {template_dir}")
         return template_dir
+
+    def write_tfvars_file(self):
+        terraform_tfvars_path = os.path.join(
+            self.get_template_dir(), self.tfvars_filename)
+        tf_var_writer = TfVarWriter(terraform_tfvars_path)
+        tf_var_writer.write_variables(self.get_terraform_variables())

cloud/shared/bin/lib/terraform.py

@@ -12,6 +12,19 @@
 from cloud.aws.templates.aws_oidc.bin.aws_cli import AwsCli
 
 
+def find_variable_default(config: ConfigLoader,
+                          variable_name: str) -> Optional[str]:
+    '''Finds the default value of a variable in the Terraform template. Does not read settings from the config file, only the Terraform template default.'''
+    with open(os.path.join(config.get_template_dir(), 'variables.tf'),
+              'r') as file:
+        content = file.read()
+    pattern = rf'variable "{variable_name}"\s*{{[^}}]*default\s*=\s*(?P<default_value>[^\n]+)'  # Barf
+    match = re.search(pattern, content, re.MULTILINE)
+    if match:
+        return match.group('default_value').strip().strip('"').strip("'")
+    return None
+
+
 def force_unlock(
         config_loader: ConfigLoader,
         lock_id: str,

cloud/shared/bin/run.py

@@ -12,7 +12,6 @@
 
 from cloud.shared.bin.lib.config_loader import ConfigLoader
 from cloud.shared.bin.lib.print import print
-from cloud.shared.bin.lib.write_tfvars import TfVarWriter
 from cloud.shared.bin.lib import backend_setup
 from cloud.shared.bin.lib import terraform
 from cloud.aws.templates.aws_oidc.bin.aws_cli import AwsCli
@@ -76,10 +75,7 @@ def main():
 
     # Write the passthrough vars to a temporary file
     print("Writing TF Vars file")
-    terraform_tfvars_path = os.path.join(
-        config.get_template_dir(), config.tfvars_filename)
-    tf_var_writter = TfVarWriter(terraform_tfvars_path)
-    tf_var_writter.write_variables(config.get_terraform_variables())
+    config.write_tfvars_file()
 
     if args.command:
         cmd = shlex.split(args.command)[0]