Flyway migration to Create Alembic with Scripts in Support (#2901)

Co-authored-by: Erik Nelsestuen <[email protected]>

db-init/src/main/resources/database/migrations/domain-cc/V2.0__Install_Alembic_Schema.sql

@@ -0,0 +1,10 @@
+
+CREATE ROLE #[alembic_username] LOGIN PASSWORD '#[alembic_password]';
+GRANT CONNECT ON DATABASE #[dbname] TO #[alembic_username];
+
+GRANT ALL PRIVILEGES ON SCHEMA #[alembic_schemaname] TO #[alembic_username];
+ALTER DEFAULT PRIVILEGES IN SCHEMA #[alembic_schemaname] GRANT ALL PRIVILEGES ON TABLES TO #[alembic_username];
+ALTER DEFAULT PRIVILEGES IN SCHEMA #[alembic_schemaname] GRANT SELECT, INSERT, UPDATE ON TABLES TO #[alembic_username];
+ALTER DEFAULT PRIVILEGES IN SCHEMA #[alembic_schemaname] GRANT ALL PRIVILEGES ON SEQUENCES TO #[alembic_username];
+ALTER DEFAULT PRIVILEGES IN SCHEMA #[alembic_schemaname] GRANT ALL PRIVILEGES ON FUNCTIONS TO #[alembic_username];
+ALTER USER #[alembic_username] SET search_path TO #[alembic_schemaname];

db-init/src/main/resources/database/migrations/domain-cc/build.gradle

@@ -0,0 +1,72 @@
+buildscript {
+    dependencies {
+        classpath 'org.postgresql:postgresql:42.6.0'
+        classpath 'org.flywaydb:flyway-database-postgresql:10.0.1'
+    }
+}
+
+plugins {
+    id 'java'
+    id "org.flywaydb.flyway" version "10.0.1"
+}
+
+ext {
+    dbCfg = [:]
+}
+
+init {
+    setDidWork(loadEnvFile())
+    println dbCfg
+
+}
+
+tasks.register('debugCfg') {}
+
+flyway {
+    defaultSchema = dbCfg.db_schema
+    driver = 'org.postgresql.Driver'
+    url = "jdbc:postgresql://${dbCfg.db_host}:${dbCfg.db_port}/postgres"
+    user = dbCfg.db_owner_username
+    password = dbCfg.db_owner_password
+    connectRetries = 10
+    initSql = "SET ROLE '${dbCfg.db_owner_username}'"
+    schemas = [dbCfg.db_schema]
+    cleanDisabled = false
+    placeholders = [
+            db_name          : dbCfg.db_name,
+            db_schema:       : 'web'
+            db_owner_username: dbCfg.db_owner_username,
+            db_rw_username   : dbCfg.db_rw_username,
+            db_rw_password   : dbCfg.db_rw_password,
+            db_ro_username   : dbCfg.db_ro_username,
+            db_ro_password   : dbCfg.db_ro_password
+    ]
+}
+
+boolean loadEnvFile() {
+    println("In loadEnvFile...")
+    try {
+        String fileName = "./do_not_commit/env/.env.${environment}"
+        File file = new File(fileName)
+        if (file.exists()) {
+            new FileReader(file).readLines().stream()
+                    .filter { it.contains('=') }
+                    .map { it.trim().split('=') }
+                    .map {fixup(it) }
+                    .forEach { dbCfg[it[0]] = it[1] }
+
+            return true
+        } else {
+            println "${fileName} doesn't exist"
+        }
+    } catch (MissingPropertyException e) {
+        println("error: ${e.class.name} - ${e.message}")
+        println "usage: ./gradlew -Penvironment=[local | dev | test | prod]> -q loadEnvFile"
+    }
+
+    return false
+}
+
+String[] fixup(String[] original) {
+    [original[0].trim().toLowerCase(), original[1].trim().replaceAll('"', '')]
+}

docker-compose.yml

@@ -143,3 +143,6 @@ services:
       FLYWAY_PLACEHOLDERS_DB_NAME: ${POSTGRES_DB}
       FLYWAY_PLACEHOLDERS_SCHEMA_NAME: ${POSTGRES_SCHEMA}
       FLYWAY_PLACEHOLDERS_USER_PASSWORD: ${POSTGRES_PASSWORD}
+      FLYWAY_PLACEHOLDERS_ALEMBIC_USERNAME: ${POSTGRES_DOMAIN_CC_USER}
+      FLYWAY_PLACEHOLDERS_ALEMBIC_SCHEMA: domain_cc
+      FLYWAY_PLACEHOLDERS_ALEMBIC_PASSWORD: ${POSTGRES_DOMAIN_CC_PW}

helm/_shared/named_templates/_postgres.tpl

@@ -60,3 +60,34 @@
       name: vro-db
       key: DB_CLIENTUSER_PASS
 {{- end }}
+
+{{- define "domainCc.alembic.envVars" -}}
+- name: DOMAIN_CC_ALEMBIC_URL
+  {{ include "vro.postgresUrl" . }}
+- name: DOMAIN_CC_USER
+  valueFrom:
+    secretKeyRef:
+      name: domain-cc-db
+      key: DOMAIN_CC_USER
+- name: DOMAIN_CC_DB
+  value: {{ .Values.global.service.db.databaseName }}
+- name: DOMAIN_CC_SCHEMA
+  value: {{ .Values.serviceNameSuffix }}
+- name: DOMAIN_CC_PW
+  valueFrom:
+    secretKeyRef:
+      name: domain-cc-db
+      key: DOMAIN_CC_PW
+- name: FLYWAY_PLACEHOLDERS_ALEMBIC_USERNAME
+  valueFrom:
+    secretKeyRef:
+      name: domain-cc-db
+      key: DOMAIN_CC_USER
+- name: FLYWAY_PLACEHOLDERS_ALEMBIC_SCHEMA
+  value: {{ .Values.serviceNameSuffix }}
+- name: FLYWAY_PLACEHOLDERS_ALEMBIC_PASSWORD
+  valueFrom:
+    secretKeyRef:
+      name: domain-cc-db
+      key: DOMAIN_CC_PW
+{{- end }}

helm/vro-app/templates/deployment.yaml

@@ -25,6 +25,7 @@ spec:
             - name: POSTGRES_DB
               value: {{ .Values.global.service.db.databaseName }}
             {{- include "vro.flyway.envVars" . | nindent 12 }}
+            {{- include "domainCc.alembic.envVars" . | nindent 12 }}
           resources:
             requests:
               cpu: 150m

scripts/create-secret.sh

@@ -0,0 +1,90 @@
+#!/usr/bin/env bash
+
+# encode string to base64
+base64_encode() {
+    echo -n "$1" | base64
+}
+
+print_usage() {
+    echo "Usage: $0 [-n, --dry-run] <env> <secret_name> KEY_1=VALUE_1 KEY_2=VALUE_2 ..."
+    echo "Options:"
+    echo "  -n, --dry-run     Preview the kubectl command without applying the Secret."
+}
+# check for dry_run flag
+dry_run=false
+while [[ "$#" -gt 0 ]]; do
+    case "$1" in
+        -n|--dry-run)
+            dry_run=true
+            shift
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+# Check for minimum number of arguments provided
+if [ "$#" -lt 3 ]; then
+    echo "Error: Insufficient arguments."
+    print_usage
+    exit 1
+fi
+
+: "${TARGET_ENV:=$1}"
+secret_name="$2"
+
+# validate target env
+case "${TARGET_ENV}" in
+  dev|qa|sandbox) choice="y"; echo "Executing $0 for env: $TARGET_ENV";;
+  prod-test|prod) read -rp "Executing $0 for env: $TARGET_ENV Please Confirm (y/n)?" choice;;
+  *)  echo "Unknown TARGET_ENV: $TARGET_ENV"
+      exit 3
+      ;;
+esac
+
+# prod environments prompt for confirmation
+case "$choice" in
+    y|Y ) echo "$TARGET_ENV confirmed: yes";;
+    * ) echo "$TARGET_ENV was not confirmed"
+        exit 4
+        ;;
+esac
+
+shift 2  # Remove the first two arguments (secret name and namespace)
+
+SECRET_MAP=""
+whitespace="  "
+# Process each key-value pair
+for arg in "$@"; do
+    if [[ "$arg" =~ ^([^=]+)=(.+)$ ]]; then
+        key="${BASH_REMATCH[1]}"
+        value="${BASH_REMATCH[2]}"
+
+        # Encode value to base64
+        encoded_value=$(base64_encode "$value")
+
+        # Append the key-value pair to YAML content
+        SECRET_MAP+="${whitespace}$key: \"$encoded_value\""
+        whitespace="\n  "
+    else
+        echo "Error: Invalid key-value pair format: $arg"
+        print_usage
+        exit 1
+    fi
+done
+
+yaml_content=$(./scripts/echo-secret-yaml.sh "$secret_name" "$SECRET_MAP")
+
+# Preview or apply the Secret YAML using kubectl
+if [ "$dry_run" = true ]; then
+    echo "Dry run: kubectl -n \"va-abd-rrd-$TARGET_ENV\" replace --force -f - <<EOF$yaml_content"
+    echo "EOF"
+else
+
+    echo "$yaml_content" | \
+        kubectl -n "va-abd-rrd-$TARGET_ENV" replace --force -f -
+    # Apply the Secret YAML using kubectl in a heredoc cat statement
+    #cat <<EOF | kubectl apply -n "$TARGET_ENV" -f -
+    #$yaml_content
+fi

scripts/setenv.sh

@@ -143,6 +143,10 @@ export POSTGRES_SCHEMA=claims
 export POSTGRES_FLYWAY_USER=vro_admin_user
 export POSTGRES_FLYWAY_PASSWORD=vro_admin_user_pw
 
+# Credentials used by Alembic to initialize domain-cc database schema
+export POSTGRES_DOMAIN_CC_USER=domain_cc_user
+export POSTGRES_DOMAIN_CC_PW=domain_cc_password
+
 # Credentials for RabbitMQ and shared across containers
 export RABBITMQ_USERNAME=user
 export RABBITMQ_PASSWORD=bitnami