Increased Cluster ram to 75% while initializing couchbase cluster

generator/deploy.sh

@@ -7,20 +7,20 @@ STACK_NAME=$2
 python generator.py parameters/${PARAMETERS_FILE}.yaml
 
 TEMPLATE_BODY="file://generated.template"
-REGION=`aws configure get region`
-
-Username="couchbase"
-Password="foo123!"
-KeyName="couchbase-${REGION}"
-License="BYOL"
-
-aws cloudformation create-stack \
---capabilities CAPABILITY_IAM \
---template-body ${TEMPLATE_BODY} \
---stack-name ${STACK_NAME} \
---region ${REGION} \
---parameters \
-ParameterKey=Username,ParameterValue=${Username} \
-ParameterKey=Password,ParameterValue=${Password} \
-ParameterKey=KeyName,ParameterValue=${KeyName} \
-ParameterKey=License,ParameterValue=${License}
+# REGION=`aws configure get region`
+#
+# Username="couchbase"
+# Password="foo123!"
+# KeyName="couchbase-${REGION}"
+# License="BYOL"
+#
+# aws cloudformation create-stack \
+# --capabilities CAPABILITY_IAM \
+# --template-body ${TEMPLATE_BODY} \
+# --stack-name ${STACK_NAME} \
+# --region ${REGION} \
+# --parameters \
+# ParameterKey=Username,ParameterValue=${Username} \
+# ParameterKey=Password,ParameterValue=${Password} \
+# ParameterKey=KeyName,ParameterValue=${KeyName} \
+# ParameterKey=License,ParameterValue=${License}

generator/generator.py

@@ -26,47 +26,63 @@ def main():
                 "Description": "Name of an existing EC2 KeyPair",
                 "Type": "AWS::EC2::KeyPair::KeyName"
             },
+            "VpcId": {
+                "Type": "AWS::EC2::VPC::Id",
+                "Description": "VpcId of your existing Virtual Private Cloud (VPC)",
+                "ConstraintDescription": "must be the VPC Id of an existing Virtual Private Cloud."
+            },
+            "Subnets": {
+               "Type": "List<AWS::EC2::Subnet::Id>",
+               "Description": "The list of PRIVATE SubnetIds in your Virtual Private Cloud (VPC)",
+               "ConstraintDescription": "must be a list of at least two existing subnets associated with at least two different availability zones. They should be residing in the selected Virtual Private Cloud."
+            },
+            "CidrIpVPC": {
+                "Type": "String",
+                "Description": "CIDR Block for existing VPC. Traffic would be allowed from this only",
+                "ConstraintDescription": "must be the VPC Id of an existing Virtual Private Cloud."
+            },
             "License": {
               "Description": "License model can be BYOL or HourlyPricing",
               "Type": "String",
-              "Default": "HourlyPricing"
+              "Default": "BYOL",
+              "ConstraintDescription": "Select BYOL only for GLP as these are pre baked images"
             }
         },
         "Mappings": {},
         "Resources": {}
     }
 
     serverVersion = parameters['serverVersion']
-    syncGatewayVersion = parameters['syncGatewayVersion']
+    # syncGatewayVersion = parameters['syncGatewayVersion']
     cluster = parameters['cluster']
 
-    template['Mappings'] = dict(template['Mappings'].items() + generateMappings(serverVersion, syncGatewayVersion).items())
+    template['Mappings'] = dict(template['Mappings'].items() + generateMappings(serverVersion).items())
     template['Resources'] = dict(template['Resources'].items() + generateMiscResources().items())
     template['Resources'] = dict(template['Resources'].items() + generateCluster(cluster).items())
 
     file = open('generated.template', 'w')
     file.write(json.dumps(template, sort_keys=True, indent=4, separators=(',', ': ')) + '\n')
     file.close()
 
-def generateMappings(serverVersion, syncGatewayVersion):
+def generateMappings(serverVersion):
     allMappings = {
         "CouchbaseServer": {
-            "4.6.4": {
-                "us-east-1": { "BYOL": "ami-63af9f19", "HourlyPricing": "ami-279faf5d" },
-                "us-east-2": { "BYOL": "ami-d17742b4", "HourlyPricing": "ami-f4704591" },
-                "us-west-1": { "BYOL": "ami-018d8061", "HourlyPricing": "ami-b88d80d8" },
-                "us-west-2": { "BYOL": "ami-eea11e96", "HourlyPricing": "ami-b99728c1" },
-                "ca-central-1": { "BYOL": "ami-4723a623", "HourlyPricing": "ami-492ca92d" },
-                "eu-central-1": { "BYOL": "ami-82079eed", "HourlyPricing": "ami-d63ca5b9" },
-                "eu-west-1": { "BYOL": "ami-1c88ef65", "HourlyPricing": "ami-fc99fe85" },
-                "eu-west-2": { "BYOL": "ami-12445e76", "HourlyPricing": "ami-cb455faf" },
-                "eu-west-3": { "BYOL": "ami-86d96ffb", "HourlyPricing": "ami-d3dd6bae" },
-                "ap-southeast-1": { "BYOL": "ami-6fed9513", "HourlyPricing": "ami-17ed956b" },
-                "ap-southeast-2": { "BYOL": "ami-3b0ef059", "HourlyPricing": "ami-c30ef0a1" },
-                "ap-south-1": { "BYOL": "ami-83b1e0ec", "HourlyPricing": "ami-c8b0e1a7" },
-                "ap-northeast-1": { "BYOL": "ami-1eea8778", "HourlyPricing": "ami-75e78a13" },
-                "ap-northeast-2": { "BYOL": "ami-b19330df", "HourlyPricing": "ami-888e2de6" },
-                "sa-east-1": { "BYOL": "ami-f4551998", "HourlyPricing": "ami-d3571bbf" }
+            "4.6.3": {
+                "us-east-1": { "BYOL": "ami-a693a3dc", "HourlyPricing": "ami-ef95a595" },
+                "us-east-2": { "BYOL": "ami-d97441bc", "HourlyPricing": "ami-62764307" },
+                "us-west-1": { "BYOL": "ami-cf8c81af", "HourlyPricing": "ami-c08c81a0" },
+                "us-west-2": { "BYOL": "ami-380dfa40", "HourlyPricing": "ami-49a11e31" },
+                "ca-central-1": { "BYOL": "ami-9822a7fc", "HourlyPricing": "ami-2e22a74a" },
+                "eu-central-1": { "BYOL": "ami-8438a1eb", "HourlyPricing": "ami-9939a0f6" },
+                "eu-west-1": { "BYOL": "ami-078aed7e", "HourlyPricing": "ami-7797f00e" },
+                "eu-west-2": { "BYOL": "ami-dd455fb9", "HourlyPricing": "ami-be7b61da" },
+                "eu-west-3": { "BYOL": "ami-d5dd6ba8", "HourlyPricing": "ami-5bc77126" },
+                "ap-southeast-1": { "BYOL": "ami-33ec944f", "HourlyPricing": "ami-13eb936f" },
+                "ap-southeast-2": { "BYOL": "ami-8910eeeb", "HourlyPricing": "ami-ec11ef8e" },
+                "ap-south-1": { "BYOL": "aami-0d8ddc62", "HourlyPricing": "ami-5db1e032" },
+                "ap-northeast-1": { "BYOL": "ami-b0e489d6", "HourlyPricing": "ami-47e48921" },
+                "ap-northeast-2": { "BYOL": "ami-ec8d2e82", "HourlyPricing": "ami-e78c2f89" },
+                "sa-east-1": { "BYOL": "ami-995519f5", "HourlyPricing": "ami-f5551999" }
             },
             "5.0.1": {
                 "us-east-1": { "BYOL": "ami-a693a3dc", "HourlyPricing": "ami-ef95a595" },
@@ -108,7 +124,7 @@ def generateMappings(serverVersion, syncGatewayVersion):
     }
     mappings = {
         "CouchbaseServer": allMappings["CouchbaseServer"][serverVersion],
-        "CouchbaseSyncGateway": allMappings["CouchbaseSyncGateway"][syncGatewayVersion]
+        # "CouchbaseSyncGateway": allMappings["CouchbaseSyncGateway"][syncGatewayVersion]
     }
     return mappings
 
@@ -136,9 +152,16 @@ def generateMiscResources():
                         "Statement": [{
                             "Effect": "Allow",
                             "Action": [
-                                "ec2:CreateTags",
-                                "ec2:DescribeInstances",
-                                "autoscaling:DescribeAutoScalingGroups"
+                                    "ec2:CreateTags",
+                                    "ec2:Describe*",
+                                    "autoscaling:DescribeAutoScalingGroups",
+                                    "cloudwatch:PutMetricData",
+                                    "cloudwatch:PutDashboard",
+                                    "cloudwatch:PutMetricAlarm",
+                                    "cloudwatch:GetMetricStatistics",
+                                    "cloudwatch:ListMetrics",
+                                    "ec2:DescribeTags",
+                                    "autoscaling:DescribeAutoScalingGroups"
                             ],
                             "Resource": "*"
                         }]
@@ -151,16 +174,19 @@ def generateMiscResources():
             "Properties": {
                 "GroupDescription" : "Enable SSH and Couchbase Ports",
                 "SecurityGroupIngress": [
-                    { "IpProtocol": "tcp", "FromPort": 22, "ToPort": 22, "CidrIp": "0.0.0.0/0" },
-                    { "IpProtocol": "tcp", "FromPort": 4369, "ToPort": 4369, "CidrIp": "0.0.0.0/0" },
-                    { "IpProtocol": "tcp", "FromPort": 4984, "ToPort": 4985, "CidrIp": "0.0.0.0/0" },
-                    { "IpProtocol": "tcp", "FromPort": 8091, "ToPort": 8094, "CidrIp": "0.0.0.0/0" },
-                    { "IpProtocol": "tcp", "FromPort": 9100, "ToPort": 9105, "CidrIp": "0.0.0.0/0" },
-                    { "IpProtocol": "tcp", "FromPort": 9998, "ToPort": 9999, "CidrIp": "0.0.0.0/0" },
-                    { "IpProtocol": "tcp", "FromPort": 11207, "ToPort": 11215, "CidrIp": "0.0.0.0/0" },
-                    { "IpProtocol": "tcp", "FromPort": 18091, "ToPort": 18093, "CidrIp": "0.0.0.0/0" },
-                    { "IpProtocol": "tcp", "FromPort": 21100, "ToPort": 21299, "CidrIp": "0.0.0.0/0" }
-                ]
+                    { "IpProtocol": "tcp", "FromPort": 22, "ToPort": 22, "CidrIp": { "Ref": "CidrIpVPC" } },
+                    { "IpProtocol": "tcp", "FromPort": 4369, "ToPort": 4369, "CidrIp": { "Ref": "CidrIpVPC" } },
+                    { "IpProtocol": "tcp", "FromPort": 4984, "ToPort": 4985, "CidrIp": { "Ref": "CidrIpVPC" } },
+                    { "IpProtocol": "tcp", "FromPort": 8091, "ToPort": 8094, "CidrIp": { "Ref": "CidrIpVPC" } },
+                    { "IpProtocol": "tcp", "FromPort": 9100, "ToPort": 9105, "CidrIp": { "Ref": "CidrIpVPC" } },
+                    { "IpProtocol": "tcp", "FromPort": 9998, "ToPort": 9999, "CidrIp": { "Ref": "CidrIpVPC" } },
+                    { "IpProtocol": "tcp", "FromPort": 11207, "ToPort": 11215, "CidrIp": { "Ref": "CidrIpVPC" } },
+                    { "IpProtocol": "tcp", "FromPort": 18091, "ToPort": 18093, "CidrIp": { "Ref": "CidrIpVPC" } },
+                    { "IpProtocol": "tcp", "FromPort": 21100, "ToPort": 21299, "CidrIp": { "Ref": "CidrIpVPC" } }
+                ],
+                "VpcId": {
+                    "Ref": "VpcId"
+                }
             }
         }
     }
@@ -194,7 +220,7 @@ def generateSyncGateway(group, rallyAutoScalingGroup):
                 "AvailabilityZones": { "Fn::GetAZs": "" },
                 "LaunchConfigurationName": { "Ref": groupName + "LaunchConfiguration" },
                 "MinSize": 0,
-                "MaxSize": 100,
+                "MaxSize": 30,
                 "DesiredCapacity": nodeCount
             }
         },
@@ -222,8 +248,8 @@ def generateSyncGateway(group, rallyAutoScalingGroup):
                             "stackName=", { "Ref": "AWS::StackName" }, "\n",
                             "baseURL=https://raw.githubusercontent.com/couchbase-partners/amazon-cloud-formation-couchbase/master/scripts/\n",
                             "wget ${baseURL}syncGateway.sh\n",
-                            "chmod +x *.sh\n",
-                            "./syncGateway.sh ${stackName}\n"
+                            "chmod +x *.sh\n"
+                            # "./syncGateway.sh ${stackName}\n"
                         ]]
                     }
                 }
@@ -247,11 +273,13 @@ def generateServer(group, rallyAutoScalingGroup):
     command = [
         "#!/bin/bash\n",
         "echo 'Running startup script...'\n",
+        "echo 'Install aws-cli...'\n"
+        "yum install -y aws-cli \n"
         "adminUsername=", { "Ref": "Username" }, "\n",
         "adminPassword=", { "Ref": "Password" }, "\n",
         "services=" + servicesParameter + "\n",
         "stackName=", { "Ref": "AWS::StackName" }, "\n",
-        "baseURL=https://raw.githubusercontent.com/couchbase-partners/amazon-cloud-formation-couchbase/master/scripts/\n",
+        "baseURL=https://raw.githubusercontent.com/GloballogicPractices/amazon-cloud-formation-couchbase/master/scripts/\n",
         "wget ${baseURL}server.sh\n",
         "wget ${baseURL}util.sh\n",
         "chmod +x *.sh\n",
@@ -268,10 +296,12 @@ def generateServer(group, rallyAutoScalingGroup):
         groupName + "AutoScalingGroup": {
             "Type": "AWS::AutoScaling::AutoScalingGroup",
             "Properties": {
-                "AvailabilityZones": { "Fn::GetAZs": "" },
+                # "VpcId" : { "Ref" : "vpcId" },
+                # "AvailabilityZones": { "Fn::GetAZs": { "Ref" : "AWS::Region" }},
+                "VPCZoneIdentifier": { "Ref": "Subnets" },
                 "LaunchConfigurationName": { "Ref": groupName + "LaunchConfiguration" },
                 "MinSize": 1,
-                "MaxSize": 100,
+                "MaxSize": 30,
                 "DesiredCapacity": nodeCount
             }
         },
@@ -280,6 +310,7 @@ def generateServer(group, rallyAutoScalingGroup):
             "Properties": {
                 "ImageId": { "Fn::FindInMap": [ "CouchbaseServer", { "Ref": "AWS::Region" }, { "Ref": "License" } ] },
                 "InstanceType": nodeType,
+                "AssociatePublicIpAddress": False,
                 "SecurityGroups": [ { "Ref": "CouchbaseSecurityGroup" } ],
                 "KeyName": { "Ref": "KeyName" },
                 "EbsOptimized": True,

generator/parameters/granular.yaml

@@ -1,32 +1,32 @@
 serverVersion: 5.0.1
-syncGatewayVersion: 1.5.1
+#syncGatewayVersion: 1.5.1
 cluster:
   - group: data
     nodeCount: 2
     nodeType: m4.4xlarge
-    dataDiskSize: 100
+    dataDiskSize: 200
     services:
       - data
   - group: query
-    nodeCount: 2
+    nodeCount: 1
     nodeType: m4.4xlarge
-    dataDiskSize: 100
+    dataDiskSize: 5
     services:
       - query
   - group: index
     nodeCount: 2
     nodeType: m4.4xlarge
-    dataDiskSize: 100
+    dataDiskSize: 200
     services:
       - index
-  - group: fts
-    nodeCount: 2
-    nodeType: m4.4xlarge
-    dataDiskSize: 100
-    services:
-      - fts
-  - group: syncgateway
-    nodeCount: 2
-    nodeType: m4.4xlarge
-    services:
-      - syncGateway
+  # - group: fts
+  #   nodeCount: 2
+  #   nodeType: m4.4xlarge
+  #   dataDiskSize: 100
+  #   services:
+  #     - fts
+  # - group: syncgateway
+  #   nodeCount: 2
+  #   nodeType: m4.4xlarge
+  #   services:
+  #     - syncGateway

generator/parameters/simple.yaml

@@ -1,12 +1,12 @@
 serverVersion: 5.0.1
-syncGatewayVersion: 1.5.1
+# syncGatewayVersion: 1.5.1
 cluster:
   - group: group1
-    nodeCount: 5
+    nodeCount: 1
     nodeType: m4.4xlarge
-    dataDiskSize: 100
+    dataDiskSize: 200
     services:
       - data
       - query
       - index
-      - fts
+      # - fts