Fix upgrade paths for different postgres versions (#321)

cloud/aws/bin/deploy.py

@@ -45,43 +45,59 @@ def _check_application_secret_length(config: ConfigLoader, aws: AwsCli):
 
 def _check_for_postgres_upgrade(config: ConfigLoader, aws: AwsCli):
  # https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_UpgradeDBInstance.PostgreSQL.html
- # For each major PG version, the oldest allowed PG minor you must be on in order to upgrade to that major version.
- # We don't really care which minor version of the upgraded version you get upgraded to, as AWS will take care of upgrading
- # the minor version later (e.g. 12.17 can only upgrade to 16.1, but then AWS will upgrade that again to 16.2
- # automatically once it's at 16.1 in the next maintenance window).
  #
- # We are currently only upgrading 12 -> 16. Fill in this table as needed for future upgrades.
+ # Because RDS won't automatically choose the appopriate minor version to upgrade to based on your current version,
+ # we have to specifically specify the valid 16.x minor upgrade path. Unfortunately, we'll have to keep the
+ # upgrade_path dictionary up to date as new minor versions of 12.x are released.
  #
- # major_version_to_upgrade_to: {current_major_version: oldest_allowable_minor_version}
- pg_upgrade_table = {16: {12: 17}}
- postgresql_major_to_apply = config.get_config_var(
- "POSTGRESQL_MAJOR_VERSION") or terraform.find_variable_default(
- config, 'postgresql_major_version')
- if postgresql_major_to_apply:
- to_apply = int(postgresql_major_to_apply)
+ # When we aren't upgrading the major version, we want to make sure POSTGRESQL_VERSION stays simply "16" so that Terraform
+ # doesn't try applying a downgrade to 16.1 if we're on 16.2 or later.
+ #
+ # If a user specifies a 16.x specifically in POSTGRESQL_VERSION, we'll pass that through directly since we assume the user
+ # knows what they are trying to do.
+
+ upgrade_path = {
+ '12.17': '16.1',
+ '12.18': '16.2',
+ }
+
+ specified_version = config.get_config_var('POSTGRESQL_VERSION') or \
+ terraform.find_variable_default(config, 'postgresql_version')
+ if specified_version:
+ major_to_apply, minor_to_apply = (
+ specified_version.split('.') + [None])[:2]
+ major_to_apply = int(major_to_apply)
  current_major, current_minor = aws.get_postgresql_version(
  f'{config.app_prefix}-{resources.DATABASE}')
- if to_apply != current_major:
+ current_version = f'{current_major}.{current_minor}'
+ if major_to_apply != current_major:
  print(
  textwrap.dedent(
  f'''
- {Color.CYAN}This version of CiviForm contains an upgrade to PostgreSQL {to_apply}. Your install is currently using PostgreSQL version {current_major}.{current_minor}.
+ {Color.CYAN}This version of CiviForm contains an upgrade to PostgreSQL {major_to_apply}. Your install is currently using PostgreSQL version {current_version}.
  
  The upgrade may take an extra 10-20 minutes to complete, during which time the CiviForm application will be unavailable. Before upgrading, ensure you have a backup of your database. You can do this by running bin/run and choosing the dumpdb command.
- Additionally, a snapshot will be performed just prior to the upgrade. The snapshot will have a name that starts with "preupgrade". You may also have a snapshot called "{config.app_prefix}-civiform-db-finalsnapshot".
- {Color.END}
+ 
+ Additionally, a snapshot will be performed just prior to the upgrade. The snapshot will have a name that starts with "preupgrade". You may also have a snapshot called "{config.app_prefix}-civiform-db-finalsnapshot".{Color.END}
  '''))
- if to_apply < current_major:
- raise ValueError(
- f'{Color.RED}Your current version of PostgreSQL appears to be newer than the version specified for this CiviForm release. Ensure you are using the correct version of the cloud-deploy-infra repo and POSTGRESQL_MAJOR_VERSION is unset or set appropriately.{Color.END}'
+ if major_to_apply < current_major:
+ print(
+ f'{Color.RED}Your current version of PostgreSQL appears to be newer than the version specified for this CiviForm release. Ensure you are using the correct version of the cloud-deploy-infra repo and POSTGRESQL_VERSION is unset or set appropriately.{Color.END}'
  )
- if to_apply not in pg_upgrade_table:
- raise ValueError(
- f'{Color.RED}Unsupported upgrade to PostgreSQL version {to_apply} specified for POSTGRESQL_MAJOR_VERSION. If this seems incorrect, contact a CiviForm maintainer.{Color.END}'
+ exit(1)
+ if major_to_apply != 16:
+ print(
+ f'{Color.RED}Unsupported upgrade to PostgreSQL version {major_to_apply} specified for POSTGRESQL_VERSION. If this seems incorrect, contact a CiviForm maintainer.{Color.END}'
  )
- if current_major not in pg_upgrade_table[to_apply]:
+ exit(1)
+ if current_major == 12 and current_minor < 17:
  print(
- f'{Color.YELLOW}This version of the deployment tool does not have information about if {current_major}.{current_minor} is sufficiently new enough to upgrade to version {to_apply}. Check https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_UpgradeDBInstance.PostgreSQL.html and verify this is a valid upgrade path.{Color.END}'
+ f'{Color.RED}In order to upgrade to PostgreSQL {major_to_apply}, you must first upgrade to PostgreSQL 12.17 or a later PostgreSQL 12 minor version. You will need to perform this upgrade in the AWS RDS console before proceeding.{Color.END}'
+ )
+ exit(1)
+ if current_version not in upgrade_path.keys():
+ print(
+ f'{Color.YELLOW}This version of the CiviForm deployment tool has no information about your current PostgreSQL version of {current_version} and thus cannot choose the correct version to upgrade to. If you proceed, we will attempt to upgrade to the latest {major_to_apply}.x version, but this may not succeed. Check https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_UpgradeDBInstance.PostgreSQL.html to verify that this is a valid upgrade path.{Color.END}'
  )
  if os.getenv('SKIP_USER_INPUT'):
  print('Proceeding since SKIP_USER_INPUT is set.')
@@ -91,21 +107,26 @@ def _check_for_postgres_upgrade(config: ConfigLoader, aws: AwsCli):
  )
  if answer.lower() not in ['y', 'yes']:
  exit(1)
- elif current_minor < pg_upgrade_table[to_apply][current_major]:
- print(
- f'{Color.RED}In order to upgrade to version {to_apply}, you must first upgrade to at least PostgreSQL {current_major}.{pg_upgrade_table[to_apply][current_major]}. You will need to perform this upgrade in the AWS RDS console before proceeding.{Color.END}'
- )
- exit(1)
+ else:
+ if minor_to_apply is None:
+ config.add_config_value(
+  'POSTGRESQL_VERSION', upgrade_path[current_version])
+
  # If a user sets ALLOW_POSTGRESQL_UPGRADE in their config file, config.get_config_var will pick it up.
  # If they've set it as an environment variable, we need to detect that and then add it to the config
  # object ourselves so that it is picked up with the manifest is compiled.
- if config.get_config_var("ALLOW_POSTGRESQL_UPGRADE") != "true":
+ if config.get_config_var('ALLOW_POSTGRESQL_UPGRADE') != 'true':
  if os.getenv('SKIP_USER_INPUT'):
- print('Proceeding since SKIP_USER_INPUT is set.')
+ print(
+ 'Proceeding with upgrade since SKIP_USER_INPUT is set.')
  else:
  answer = input(
  f'{Color.YELLOW}Would you like to proceed with the upgrade? (y/N): {Color.END}'
  )
  if answer.lower() not in ['y', 'yes']:
  exit(2)
- config.add_config_value("ALLOW_POSTGRESQL_UPGRADE", "true")
+ config.add_config_value('ALLOW_POSTGRESQL_UPGRADE', 'true')
+
+ print(
+ f'{Color.CYAN}Proceeding with upgrade to PostgreSQL {config.get_config_var("POSTGRESQL_VERSION")}.{Color.END}'
+ )

cloud/aws/templates/aws_oidc/main.tf

@@ -15,7 +15,7 @@ resource "aws_db_parameter_group" "civiform" {
  Type = "Civiform DB Parameters"
  }
 
- family = "postgres${var.postgresql_major_version}"
+ family = "postgres${split(".", var.postgresql_version)[0]}"
 
  parameter {
  name = "log_connections"
@@ -46,7 +46,7 @@ resource "aws_db_instance" "civiform" {
  storage_throughput = var.aws_db_storage_throughput
  iops = var.aws_db_iops
  engine = "postgres"
- engine_version = var.postgresql_major_version
+ engine_version = var.postgresql_version
  allow_major_version_upgrade = var.allow_postgresql_upgrade
  username = aws_secretsmanager_secret_version.postgres_username_secret_version.secret_string
  password = aws_secretsmanager_secret_version.postgres_password_secret_version.secret_string

cloud/aws/templates/aws_oidc/variable_definitions.json

@@ -393,10 +393,10 @@
  "tfvar": true,
  "type": "bool"
  },
- "POSTGRESQL_MAJOR_VERSION": {
+ "POSTGRESQL_VERSION": {
  "required": false,
  "secret": false,
  "tfvar": true,
- "type": "integer"
+ "type": "string"
  }
 }

cloud/aws/templates/aws_oidc/variables.tf

@@ -502,8 +502,8 @@ variable "allow_postgresql_upgrade" {
  default = false
 }
 
-variable "postgresql_major_version" {
- type = number
- description = "Major version of PostgreSQL to use"
- default = 16
+variable "postgresql_version" {
+ type = string
+ description = "Version of PostgreSQL to use. When set to only the major version, picks the latest minor version. Otherwise, deploys exactly the version specified."
+ default = "16"
 }