ASGARD-1011 - Support RDS in VPC

grails-app/controllers/com/netflix/asgard/AutoScalingController.groovy

@@ -37,6 +37,7 @@ import com.netflix.asgard.model.GroupedInstance
 import com.netflix.asgard.model.InstancePriceType
 import com.netflix.asgard.model.SubnetTarget
 import com.netflix.asgard.model.Subnets
+import com.netflix.asgard.push.GroupDeleteOperation
 import com.netflix.grails.contextParam.ContextParam
 import grails.converters.JSON
 import grails.converters.XML
@@ -57,6 +58,7 @@ class AutoScalingController {
     def configService
     def instanceTypeService
     def mergedInstanceService
+    def pushService
     def spotInstanceRequestService
     def stackService
 
@@ -449,25 +451,12 @@ class AutoScalingController {
         UserContext userContext = UserContext.of(request)
         String name = params.name
         AutoScalingGroup group = awsAutoScalingService.getAutoScalingGroup(userContext, name)
-        Boolean showGroupNext = false
         if (!group) {
             flash.message = "Auto Scaling Group '${name}' not found."
         } else {
-            if (group?.instances?.size() <= 0) {
-                try {
-                    awsAutoScalingService.deleteAutoScalingGroup(userContext, name)
-                    flash.message = "AutoScaling Group '${name}' has been deleted."
-                } catch (Exception e) {
-                    flash.message = "Could not delete Auto Scaling Group: ${e}"
-                    showGroupNext = true
-                }
-            } else {
-                flash.message = "You cannot delete an auto scaling group that still has instances. " +
-                        "Set the min and max to 0, wait for the instances to disappear, then try deleting again."
-                showGroupNext = true
-            }
+            GroupDeleteOperation operation = pushService.startGroupDelete(userContext, group)
+            redirect(controller: 'task', action: 'show', params: [id: operation.taskId])
         }
-        showGroupNext ? redirect(action: 'show', params: [id: name]) : redirect(action: 'list')
     }
 
     def postpone = {

grails-app/controllers/com/netflix/asgard/RdsInstanceController.groovy

@@ -15,9 +15,13 @@
  */
 package com.netflix.asgard
 
+import com.amazonaws.services.ec2.model.SecurityGroup
 import com.amazonaws.services.rds.model.DBInstance
 import com.amazonaws.services.rds.model.DBSnapshot
+import com.amazonaws.services.rds.model.DBSubnetGroup
 import com.netflix.asgard.AwsRdsService.Engine
+import com.netflix.asgard.model.SubnetTarget
+import com.netflix.asgard.model.Subnets
 import com.netflix.grails.contextParam.ContextParam
 import grails.converters.JSON
 import grails.converters.XML
@@ -56,11 +60,23 @@ class RdsInstanceController {
 
     def create = {
         UserContext userContext = UserContext.of(request)
+        List<SecurityGroup> effectiveGroups = awsEc2Service.getEffectiveSecurityGroups(userContext).sort {
+            it.groupName?.toLowerCase()
+        }
+        Map<Object, List<SecurityGroup>> securityGroupsGroupedByVpcId = effectiveGroups.groupBy { it.vpcId }
+        Subnets subnets = awsEc2Service.getSubnets(userContext)
+        Map<String, String> purposeToVpcId = subnets.mapPurposeToVpcId()
         [
             'allDBSecurityGroups' : awsRdsService.getDBSecurityGroups(userContext),
             'allDBInstanceClasses' : AwsRdsService.getDBInstanceClasses(),
             'allDbInstanceEngines' : Engine.values()*.awsValue,
             'allLicenseModels' : AwsRdsService.getLicenseModels(),
+            'purposeToVpcId': purposeToVpcId,
+            'securityGroupsGroupedByVpcId': securityGroupsGroupedByVpcId,
+            'selectedSecurityGroups': Requests.ensureList(params.selectedSecurityGroups),
+            'subnetPurpose': params.subnetPurpose ?: null,
+            'subnetPurposes': subnets.getPurposesForZones(awsEc2Service.getAvailabilityZones(userContext)*.zoneName, SubnetTarget.ELB).sort(),
+            'vpcId': purposeToVpcId[params.subnetPurpose],
             'zoneList':awsEc2Service.getAvailabilityZones(userContext)
         ]
     }
@@ -73,26 +89,40 @@ class RdsInstanceController {
         } else {
             try {
                 boolean multiAZ = "on".equals(params.multiAZ)
-                def selectedDBSecurityGroups = (params.selectedDBSecurityGroups instanceof String) ? [params.selectedDBSecurityGroups] : params.selectedDBSecurityGroups as List
-                if (!selectedDBSecurityGroups) selectedDBSecurityGroups = ["default"]
-                //awsRdsService.createDBSecurityGroup(params.name, params.description)
+                Subnets subnets = awsEc2Service.getSubnets(userContext)
+                List<String> selectedSecurityGroups = Requests.ensureList(params.selectedSecurityGroups)
+                List<String> selectedDBSecurityGroups = Requests.ensureList(params.selectedDBSecurityGroups)
+                if (!selectedDBSecurityGroups && !params.subnetPurpose) {
+                    selectedDBSecurityGroups = ["default"]
+                }
+
+                final DBSubnetGroup dbSubnetGroup = new DBSubnetGroup()
+                    .withDBSubnetGroupName(params.dBName)
+                    .withDBSubnetGroupDescription(params.dBName)
+                    .withSubnets(subnets.allSubnets)
+                    .withVpcId(subnets.mapPurposeToVpcId()[params.subnetPurpose])
 
                 final DBInstance dbInstance = new DBInstance()
                     .withAllocatedStorage(params.allocatedStorage as Integer)
-                    .withAvailabilityZone(params.availabilityZone)
                     .withBackupRetentionPeriod(params.backupRetentionPeriod as Integer)
-                    .withDBInstanceClass(params.dBInstanceClass,)
+                    .withDBInstanceClass(params.dBInstanceClass)
                     .withDBInstanceIdentifier(params.dBInstanceIdentifier)
                     .withDBName(params.dBName)
+                    .withDBSubnetGroup(dbSubnetGroup)
                     .withEngine(params.engine)
                     .withDBSecurityGroups(selectedDBSecurityGroups)
                     .withMasterUsername(params.masterUsername)
                     .withMultiAZ(multiAZ)
                     .withPreferredBackupWindow(params.preferredBackupWindow)
                     .withPreferredMaintenanceWindow(params.preferredMaintenanceWindow)
                     .withLicenseModel(params.licenseModel)
+                    .withVpcSecurityGroups(selectedSecurityGroups)
 
-                awsRdsService.createDBInstance(userContext, dbInstance, params.masterUserPassword, params.port as Integer)
+                if (!multiAZ) {
+                    dbInstance.availabilityZone = params.availabilityZone
+                }
+
+                awsRdsService.createDBInstance(userContext, dbInstance, params.masterUserPassword, params.port as Integer, params.subnetPurpose ?: null)
                 flash.message = "DB Instance '${params.dBInstanceIdentifier}' has been created."
                 redirect(action: 'show', params: [id: params.dBInstanceIdentifier])
             } catch (Exception e) {
@@ -147,7 +177,15 @@ class RdsInstanceController {
         if (!dbInstance) {
             Requests.renderNotFound('DB Instance', dbInstanceId, this)
         } else {
-            def details = ['dbInstance':dbInstance, 'snapshots' : snapshots]
+            List<String> subnetIds = dbInstance.DBSubnetGroup?.subnets?.collect { it.subnetIdentifier } ?: []
+
+            def details = [
+                'dbInstance': dbInstance,
+                'snapshots': snapshots,
+                'vpcId': awsEc2Service.getSubnets(userContext).coerceLoneOrNoneFromIds(subnetIds)?.vpcId ?: '',
+                'subnets': subnetIds,
+                'vpcSecurityGroupsIds': dbInstance.vpcSecurityGroups.collect { it.vpcSecurityGroupId }
+            ]
             withFormat {
                 html { return details }
                 xml { new XML(details).render(response) }
@@ -180,10 +218,13 @@ class DbCreateCommand {
     String dBInstanceClass // Valid values: db.m1.small | db.m1.large | db.m1.xlarge | db.m2.2xlarge | db.m2.4xlarge
     String dBInstanceIdentifier // Constraints: Must contain from 1 to 63 alphanumeric characters or hyphens. First character must be a letter. Cannot end with a hyphen or contain two consecutive hyphens.
     String dBName // Cannot be empty. Must contain 1 to 64 alphanumeric characters. Cannot be a word reserved by the specified database engine.
+    Collection<String> selectedSecurityGroups // At least one
     Collection<String> selectedDBSecurityGroups // At least one
     String masterUsername // Must be an alphanumeric string containing from 1 to 16 characters.
     String masterUserPassword // Must contain 4 to 16 alphanumeric characters.
     Integer port
+    String multiAZ
+    String subnetPurpose
     String preferredBackupWindow // Constraints: Must be in the format hh24:mi-hh24:mi.
     // Times should be 24-hour Universal Time Coordinated (UTC).
     // Must not conflict with the --preferred-maintenance-window.
@@ -196,12 +237,34 @@ class DbCreateCommand {
 
     static constraints = {
         allocatedStorage(nullable: false, range: 5..1024)
-        availabilityZone(nullable: false, blank: false) // Need more -- custom validator?
+        availabilityZone(nullable: false, validator: { value, object -> "on".equals(object.multiAZ) != value.length() > 0 ?: 'dbCreateCommand.multiaz.availabilityzones.error' })
         backupRetentionPeriod(blank: false, nullable: false, range: 0..8)
         dBInstanceClass(nullable: false, blank: false)
         dBInstanceIdentifier(nullable: false, blank: false, size: 1..63, matches: '[a-zA-Z]{1}[a-zA-Z0-9-]*[^-]$') // This match does not check the double-hyphen
         dBName(nullable: false, blank: false, size: 1..64, matches: '[a-zA-Z0-9]{1,64}')
-        selectedDBSecurityGroups(nullable: false, minSize: 1)
+        selectedSecurityGroups(validator: {
+            value, object ->
+                if (object.subnetPurpose) {
+                    if (!value && !object.selectedDBSecurityGroups) {
+                        'dbCreateCommand.selectedSecurityGroups.minSize.error'
+                    }
+                } else {
+                    if (value) {
+                        'dbCreateCommand.selectedSecurityGroups.vpc.error'
+                    }
+                }
+            })
+        selectedDBSecurityGroups(validator: {
+            value, object ->
+                if (object.subnetPurpose) {
+                    if (!value && !object.selectedSecurityGroups) {
+                        'dbCreateCommand.selectedDBSecurityGroups.minSize.error'
+                    }
+                    if (value && object) {
+                        'dbCreateCommand.selectedDBSecurityGroups.vpc.error'
+                    }
+                }
+            })
         masterUsername(nullable: false, blank: false, size: 1..16, matches: '[a-zA-Z0-9]{1,16}')
         masterUserPassword(nullable: false, blank: false, size: 4..16, matches: '[a-zA-Z0-9]{4,16}')
         port(nullable: false)

grails-app/i18n/messages.properties

@@ -81,6 +81,7 @@ dbCreateCommand.dBName.matches.error=DB Name must be 1 to 64 alphanumeric charac
 dbCreateCommand.selectedDBSecurityGroups.nullable.error=At least one DB Security Group must be selected
 dbCreateCommand.selectedDBSecurityGroups.blank.error=At least one DB Security Group must be selected
 dbCreateCommand.selectedDBSecurityGroups.minSize.error=At least one DB Security Group must be selected
+dbCreateCommand.selectedDBSecurityGroups.vpc.error=DB Security Group cannot be selected for RDS in VPC
 dbCreateCommand.masterUsername.nullable.error=Master username must be 1 to 16 alphanumeric characters
 dbCreateCommand.masterUsername.blank.error=Master username must be 1 to 16 alphanumeric characters
 dbCreateCommand.masterUsername.size.error=Master username must be 1 to 16 alphanumeric characters
@@ -92,6 +93,9 @@ dbCreateCommand.masterUserPassword.matches.error=Master user password must be 4
 dbCreateCommand.port.blank.error=Port must not be blank
 dbCreateCommand.port.nullable.error=Port must not be blank
 dbCreateCommand.preferredBackupWindow.matches.error=If specified, preferred backup window must be in the format hh24:mi-hh24:mi, 24-hour Universal Time Coordinated (UTC), must not conflict with the preferred maintenance window, must be at least 2 hours, and cannot be set if the backup retention period has not been specified.
+dbCreateCommand.multiaz.availabilityzones.error=Either specify multiple AZ, or select an availability zone
+dbCreateCommand.selectedSecurityGroups.vpc.error=Select only DB Security Groups for non-VPC RDS
+dbCreateCommand.selectedSecurityGroups.minSize.error=At least one Security Group must be selected
 
 dbUpdateCommand.allocatedStorage.nullable.error=Allocated storage must be a number between 5 and 1024
 dbUpdateCommand.allocatedStorage.blank.error=Allocated storage must be a number between 5 and 1024
@@ -102,6 +106,7 @@ dbUpdateCommand.dBInstanceClass.nullable.error=DB Instance class must not be bla
 dbUpdateCommand.selectedDBSecurityGroups.nullable.error=At least one DB Security Group must be selected
 dbUpdateCommand.selectedDBSecurityGroups.blank.error=At least one DB Security Group must be selected
 dbUpdateCommand.selectedDBSecurityGroups.minSize.error=At least one DB Security Group must be selected
+dbUpdateCommand.selectedSecurityGroups.minSize.error=At least one Security Group must be selected
 dbUpdateCommand.masterUserPassword.size.error=If specified, master user password must be 4 to 16 alphanumeric characters
 dbUpdateCommand.masterUserPassword.matches.error=If specified, master user password must be 4 to 16 alphanumeric characters
 dbUpdateCommand.preferredBackupWindow.matches.error=If specified, preferred backup window must be in the format hh24:mi-hh24:mi, 24-hour Universal Time Coordinated (UTC), must not conflict with the preferred maintenance window, must be at least 2 hours, and cannot be set if the backup retention period has not been specified.

grails-app/services/com/netflix/asgard/AwsRdsService.groovy

@@ -21,6 +21,7 @@ import com.amazonaws.services.rds.model.AuthorizeDBSecurityGroupIngressRequest
 import com.amazonaws.services.rds.model.CreateDBInstanceRequest
 import com.amazonaws.services.rds.model.CreateDBSecurityGroupRequest
 import com.amazonaws.services.rds.model.CreateDBSnapshotRequest
+import com.amazonaws.services.rds.model.CreateDBSubnetGroupRequest
 import com.amazonaws.services.rds.model.DBInstance
 import com.amazonaws.services.rds.model.DBSecurityGroup
 import com.amazonaws.services.rds.model.DBSnapshot
@@ -31,6 +32,8 @@ import com.amazonaws.services.rds.model.DescribeDBInstancesRequest
 import com.amazonaws.services.rds.model.DescribeDBSecurityGroupsRequest
 import com.amazonaws.services.rds.model.DescribeDBSnapshotsRequest
 import com.amazonaws.services.rds.model.DescribeDBSnapshotsResult
+import com.amazonaws.services.rds.model.DescribeDBSubnetGroupsRequest
+import com.amazonaws.services.rds.model.DescribeDBSubnetGroupsResult
 import com.amazonaws.services.rds.model.ModifyDBInstanceRequest
 import com.amazonaws.services.rds.model.RestoreDBInstanceFromDBSnapshotRequest
 import com.amazonaws.services.rds.model.RevokeDBSecurityGroupIngressRequest
@@ -115,13 +118,34 @@ class AwsRdsService implements CacheInitializer, InitializingBean {
         }, Link.to(EntityType.rdsInstance, dbInstanceId))
     }
 
-    DBInstance createDBInstance(UserContext userContext, DBInstance templateDbInstance, String masterUserPassword, Integer port) {
+    DBInstance createDBInstance(UserContext userContext, DBInstance templateDbInstance, String masterUserPassword, Integer port, String subnetPurpose) {
         final BeanState templateDbInstanceState = BeanState.ofSourceBean(templateDbInstance)
         final CreateDBInstanceRequest request = templateDbInstanceState.injectState(new CreateDBInstanceRequest())
         request.masterUserPassword = masterUserPassword
         if (port) {request.setPort(port)}
         taskService.runTask(userContext, "Creating DB instance '${templateDbInstance.DBInstanceIdentifier}'", { task ->
-            final DBInstance createdInstance = awsClient.by(userContext.region).createDBInstance(request)
+            if (subnetPurpose) {
+                DescribeDBSubnetGroupsResult dbSubnetGroups
+                try {
+                    DescribeDBSubnetGroupsRequest subnetExistsRequest = new DescribeDBSubnetGroupsRequest()
+                        .withDBSubnetGroupName(templateDbInstance.DBSubnetGroup.DBSubnetGroupName)
+                    dbSubnetGroups = awsClient.by(userContext.region).describeDBSubnetGroups(subnetExistsRequest)?.DBSubnetGroups
+                } catch (AmazonServiceException ignored) {
+                    dbSubnetGroups = null
+                }
+
+                if (!dbSubnetGroups) {
+                    CreateDBSubnetGroupRequest subnetRequest = new CreateDBSubnetGroupRequest()
+                        .withDBSubnetGroupName(templateDbInstance.DBSubnetGroup.DBSubnetGroupName)
+                        .withDBSubnetGroupDescription(templateDbInstance.DBSubnetGroup.DBSubnetGroupDescription)
+                        .withSubnetIds(templateDbInstance.DBSubnetGroup.subnets.collect { it.subnetId })
+                    awsClient.by(userContext.region).createDBSubnetGroup(subnetRequest)
+                }
+                // If this is a VPC RDS then we must find the proper DBSubnetGroupName.
+                request.DBSubnetGroupName = templateDbInstance.DBSubnetGroup.DBSubnetGroupName
+                request.vpcSecurityGroupIds = templateDbInstance.vpcSecurityGroups
+            }
+            DBInstance createdInstance = awsClient.by(userContext.region).createDBInstance(request)
             caches.allDBInstances.by(userContext.region).put(createdInstance.getDBInstanceIdentifier(), createdInstance)
         }, Link.to(EntityType.rdsInstance, templateDbInstance.DBInstanceIdentifier))
         getDBInstance(userContext, templateDbInstance.DBInstanceIdentifier)

grails-app/views/rdsInstance/create.gsp

@@ -93,7 +93,9 @@
             <td class="name" title="Multiple AZ"><label for="multiAZ">${params.multiAZ}  Multiple AZ:</label></td>
             <td class="value"><g:checkBox name="multiAZ" value="on" checked="${'on' == params.multiAZ }"/></td>
           </tr>
-          <tr class="prop">
+          <g:render template="/common/vpcSelection" model="[awsAction: 'Create', awsObject: 'RDS']"/>
+          <g:render template="/common/securityGroupSelection" />
+          <tr class="prop" name="dbSecurityGroups">
             <td class="name">DB Security Groups:</td>
             <td class="value">
               <table>

grails-app/views/rdsInstance/list.gsp

@@ -57,7 +57,10 @@
               <td>${dbi.dBInstanceStatus}</td>
               <td>${dbi.dBName}</td>
               <td>${dbi.dBInstanceClass}</td>
-              <td>${dbi.availabilityZone}</td>
+              <td>
+                <g:if test="${dbi.multiAZ}">All</g:if>
+                <g:else>${dbi.availabilityZone}</g:else>
+              </td>
             </tr>
           </g:each>
           </tbody>

grails-app/views/rdsInstance/show.gsp

@@ -97,6 +97,14 @@
           <td class="name">DB Engine Version:</td>
           <td class="value">${dbInstance.engineVersion}</td>
         </tr>
+        <tr class="prop">
+          <td class="name">VPC:</td>
+          <td class="value">${vpcId}</td>
+        </tr>
+        <tr class="prop">
+          <td class="name">Subnets:</td>
+          <td class="value">${subnets}</td>
+        </tr>
         <tr class="prop">
           <td class="name" title="Creation Date">DB Create Date:</td>
           <td class="value"><g:formatDate date="${dbInstance.instanceCreateTime}"/></td>
@@ -133,6 +141,10 @@
             </table>
           </td>
         </tr>
+        <tr class="prop">
+          <td class="name">VPC Security Groups:</td>
+          <td class="value">${vpcSecurityGroupsIds}</td>
+        </tr>
         <tr class="prop">
           <td class="name">DB Snapshots:</td>
           <td class="value">