From a5c8cb58ac9714ca7abbe4681251effec87e4483 Mon Sep 17 00:00:00 2001 From: faizan Date: Sat, 24 Feb 2018 18:53:25 +0530 Subject: [PATCH 01/52] Custom code --- generator/deploy.sh | 34 ++++---- generator/generator.py | 129 ++++++++++++++++------------- generator/parameters/granular.yaml | 30 +++---- 3 files changed, 103 insertions(+), 90 deletions(-) diff --git a/generator/deploy.sh b/generator/deploy.sh index f904737..3d4fd2f 100755 --- a/generator/deploy.sh +++ b/generator/deploy.sh @@ -7,20 +7,20 @@ STACK_NAME=$2 python generator.py parameters/${PARAMETERS_FILE}.yaml TEMPLATE_BODY="file://generated.template" -REGION=`aws configure get region` - -Username="couchbase" -Password="foo123!" -KeyName="couchbase-${REGION}" -License="BYOL" - -aws cloudformation create-stack \ ---capabilities CAPABILITY_IAM \ ---template-body ${TEMPLATE_BODY} \ ---stack-name ${STACK_NAME} \ ---region ${REGION} \ ---parameters \ -ParameterKey=Username,ParameterValue=${Username} \ -ParameterKey=Password,ParameterValue=${Password} \ -ParameterKey=KeyName,ParameterValue=${KeyName} \ -ParameterKey=License,ParameterValue=${License} +# REGION=`aws configure get region` +# +# Username="couchbase" +# Password="foo123!" +# KeyName="couchbase-${REGION}" +# License="BYOL" +# +# aws cloudformation create-stack \ +# --capabilities CAPABILITY_IAM \ +# --template-body ${TEMPLATE_BODY} \ +# --stack-name ${STACK_NAME} \ +# --region ${REGION} \ +# --parameters \ +# ParameterKey=Username,ParameterValue=${Username} \ +# ParameterKey=Password,ParameterValue=${Password} \ +# ParameterKey=KeyName,ParameterValue=${KeyName} \ +# ParameterKey=License,ParameterValue=${License} diff --git a/generator/generator.py b/generator/generator.py index 9ec5df9..5eb5242 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -26,21 +26,33 @@ def main(): "Description": "Name of an existing EC2 KeyPair", "Type": "AWS::EC2::KeyPair::KeyName" }, - "License": { - "Description": "License model can be BYOL or HourlyPricing", - "Type": "String", - "Default": "HourlyPricing" - } + "VpcId": { + "Type": "AWS::EC2::VPC::Id", + "Description": "VpcId of your existing Virtual Private Cloud (VPC)", + "ConstraintDescription": "must be the VPC Id of an existing Virtual Private Cloud." + }, + "CidrIpVPC": { + "Type": "AWS::EC2::VPCCidrBlock", + "Description": "CIDR Block for existing VPC. Traffic would be allowed from this only", + "ConstraintDescription": "must be the VPC Id of an existing Virtual Private Cloud." + }, + "Region": { + "Type": "AWS::Region", + "Description": "Region where stack is being deployed", + #"ConstraintDescription": "must be a list of at least two existing subnets associated with at least two different availability zones. They should be residing in the selected Virtual Private Cloud." + "default": "us-west-2" + }, + }, "Mappings": {}, "Resources": {} } serverVersion = parameters['serverVersion'] - syncGatewayVersion = parameters['syncGatewayVersion'] + # syncGatewayVersion = parameters['syncGatewayVersion'] cluster = parameters['cluster'] - template['Mappings'] = dict(template['Mappings'].items() + generateMappings(serverVersion, syncGatewayVersion).items()) + template['Mappings'] = dict(template['Mappings'].items() + generateMappings(serverVersion).items()) template['Resources'] = dict(template['Resources'].items() + generateMiscResources().items()) template['Resources'] = dict(template['Resources'].items() + generateCluster(cluster).items()) @@ -48,25 +60,25 @@ def main(): file.write(json.dumps(template, sort_keys=True, indent=4, separators=(',', ': ')) + '\n') file.close() -def generateMappings(serverVersion, syncGatewayVersion): +def generateMappings(serverVersion): allMappings = { "CouchbaseServer": { - "4.6.4": { - "us-east-1": { "BYOL": "ami-63af9f19", "HourlyPricing": "ami-279faf5d" }, - "us-east-2": { "BYOL": "ami-d17742b4", "HourlyPricing": "ami-f4704591" }, - "us-west-1": { "BYOL": "ami-018d8061", "HourlyPricing": "ami-b88d80d8" }, - "us-west-2": { "BYOL": "ami-eea11e96", "HourlyPricing": "ami-b99728c1" }, - "ca-central-1": { "BYOL": "ami-4723a623", "HourlyPricing": "ami-492ca92d" }, - "eu-central-1": { "BYOL": "ami-82079eed", "HourlyPricing": "ami-d63ca5b9" }, - "eu-west-1": { "BYOL": "ami-1c88ef65", "HourlyPricing": "ami-fc99fe85" }, - "eu-west-2": { "BYOL": "ami-12445e76", "HourlyPricing": "ami-cb455faf" }, - "eu-west-3": { "BYOL": "ami-86d96ffb", "HourlyPricing": "ami-d3dd6bae" }, - "ap-southeast-1": { "BYOL": "ami-6fed9513", "HourlyPricing": "ami-17ed956b" }, - "ap-southeast-2": { "BYOL": "ami-3b0ef059", "HourlyPricing": "ami-c30ef0a1" }, - "ap-south-1": { "BYOL": "ami-83b1e0ec", "HourlyPricing": "ami-c8b0e1a7" }, - "ap-northeast-1": { "BYOL": "ami-1eea8778", "HourlyPricing": "ami-75e78a13" }, - "ap-northeast-2": { "BYOL": "ami-b19330df", "HourlyPricing": "ami-888e2de6" }, - "sa-east-1": { "BYOL": "ami-f4551998", "HourlyPricing": "ami-d3571bbf" } + "4.6.3": { + "us-east-1": "ami-9e87f2e4", + "us-east-2": "ami-9e87f2e4", + "us-west-1": "ami-9e87f2e4", + "us-west-2": "ami-380dfa40", + "ca-central-1": "ami-492ca92d", + "eu-central-1": "ami-d63ca5b9", + "eu-west-1": "ami-fc99fe85", + "eu-west-2": "ami-cb455faf", + "eu-west-3": "ami-d3dd6bae", + "ap-southeast-1": "ami-17ed956b", + "ap-southeast-2": "ami-c30ef0a1", + "ap-south-1": "ami-c8b0e1a7", + "ap-northeast-1": "ami-75e78a13", + "ap-northeast-2": "ami-888e2de6", + "sa-east-1": "ami-d3571bbf" }, "5.0.1": { "us-east-1": { "BYOL": "ami-a693a3dc", "HourlyPricing": "ami-ef95a595" }, @@ -86,29 +98,29 @@ def generateMappings(serverVersion, syncGatewayVersion): "sa-east-1": { "BYOL": "ami-995519f5", "HourlyPricing": "ami-f5551999" } } }, - "CouchbaseSyncGateway": { - "1.5.1": { - "us-east-1": { "BYOL": "ami-8294a4f8", "HourlyPricing": "ami-6d93a317" }, - "us-east-2": { "BYOL": "ami-0877426d", "HourlyPricing": "ami-0f75406a" }, - "us-west-1": { "BYOL": "ami-288c8148", "HourlyPricing": "ami-76f2ff16" }, - "us-west-2": { "BYOL": "ami-589c2320", "HourlyPricing": "ami-41a01f39" }, - "ca-central-1": { "BYOL": "ami-ad20a5c9", "HourlyPricing": "ami-c22da8a6" }, - "eu-central-1": { "BYOL": "ami-103aa37f", "HourlyPricing": "ami-d5069fba" }, - "eu-west-1": { "BYOL": "ami-b696f1cf", "HourlyPricing": "ami-c293f4bb" }, - "eu-west-2": { "BYOL": "ami-6c445e08", "HourlyPricing": "ami-11445e75" }, - "eu-west-3": { "BYOL": "ami-c9d86eb4", "HourlyPricing": "ami-58c77125" }, - "ap-southeast-1": { "BYOL": "ami-10eb936c", "HourlyPricing": "ami-a4ed95d8" }, - "ap-southeast-2": { "BYOL": "ami-a610eec4", "HourlyPricing": "ami-2911ef4b" }, - "ap-south-1": { "BYOL": "ami-898cdde6", "HourlyPricing": "ami-058cdd6a" }, - "ap-northeast-1": { "BYOL": "ami-b9e588df", "HourlyPricing": "ami-b3e588d5" }, - "ap-northeast-2": { "BYOL": "ami-38933056", "HourlyPricing": "ami-648c2f0a" }, - "sa-east-1": { "BYOL": "ami-3654185a", "HourlyPricing": "ami-b95a16d5" } - } - } + # "CouchbaseSyncGateway": { + # "1.5.1": { + # "us-east-1": { "BYOL": "ami-8294a4f8", "HourlyPricing": "ami-6d93a317" }, + # "us-east-2": { "BYOL": "ami-0877426d", "HourlyPricing": "ami-0f75406a" }, + # "us-west-1": { "BYOL": "ami-288c8148", "HourlyPricing": "ami-76f2ff16" }, + # "us-west-2": { "BYOL": "ami-589c2320", "HourlyPricing": "ami-41a01f39" }, + # "ca-central-1": { "BYOL": "ami-ad20a5c9", "HourlyPricing": "ami-c22da8a6" }, + # "eu-central-1": { "BYOL": "ami-103aa37f", "HourlyPricing": "ami-d5069fba" }, + # "eu-west-1": { "BYOL": "ami-b696f1cf", "HourlyPricing": "ami-c293f4bb" }, + # "eu-west-2": { "BYOL": "ami-6c445e08", "HourlyPricing": "ami-11445e75" }, + # "eu-west-3": { "BYOL": "ami-c9d86eb4", "HourlyPricing": "ami-58c77125" }, + # "ap-southeast-1": { "BYOL": "ami-10eb936c", "HourlyPricing": "ami-a4ed95d8" }, + # "ap-southeast-2": { "BYOL": "ami-a610eec4", "HourlyPricing": "ami-2911ef4b" }, + # "ap-south-1": { "BYOL": "ami-898cdde6", "HourlyPricing": "ami-058cdd6a" }, + # "ap-northeast-1": { "BYOL": "ami-b9e588df", "HourlyPricing": "ami-b3e588d5" }, + # "ap-northeast-2": { "BYOL": "ami-38933056", "HourlyPricing": "ami-648c2f0a" }, + # "sa-east-1": { "BYOL": "ami-3654185a", "HourlyPricing": "ami-b95a16d5" } + # } + # } } mappings = { "CouchbaseServer": allMappings["CouchbaseServer"][serverVersion], - "CouchbaseSyncGateway": allMappings["CouchbaseSyncGateway"][syncGatewayVersion] + # "CouchbaseSyncGateway": allMappings["CouchbaseSyncGateway"][syncGatewayVersion] } return mappings @@ -151,15 +163,15 @@ def generateMiscResources(): "Properties": { "GroupDescription" : "Enable SSH and Couchbase Ports", "SecurityGroupIngress": [ - { "IpProtocol": "tcp", "FromPort": 22, "ToPort": 22, "CidrIp": "0.0.0.0/0" }, - { "IpProtocol": "tcp", "FromPort": 4369, "ToPort": 4369, "CidrIp": "0.0.0.0/0" }, - { "IpProtocol": "tcp", "FromPort": 4984, "ToPort": 4985, "CidrIp": "0.0.0.0/0" }, - { "IpProtocol": "tcp", "FromPort": 8091, "ToPort": 8094, "CidrIp": "0.0.0.0/0" }, - { "IpProtocol": "tcp", "FromPort": 9100, "ToPort": 9105, "CidrIp": "0.0.0.0/0" }, - { "IpProtocol": "tcp", "FromPort": 9998, "ToPort": 9999, "CidrIp": "0.0.0.0/0" }, - { "IpProtocol": "tcp", "FromPort": 11207, "ToPort": 11215, "CidrIp": "0.0.0.0/0" }, - { "IpProtocol": "tcp", "FromPort": 18091, "ToPort": 18093, "CidrIp": "0.0.0.0/0" }, - { "IpProtocol": "tcp", "FromPort": 21100, "ToPort": 21299, "CidrIp": "0.0.0.0/0" } + { "IpProtocol": "tcp", "FromPort": 22, "ToPort": 22, "CidrIp": { "Ref": "CidrIpVPC" } }, + { "IpProtocol": "tcp", "FromPort": 4369, "ToPort": 4369, "CidrIp": { "Ref": "CidrIpVPC" } }, + { "IpProtocol": "tcp", "FromPort": 4984, "ToPort": 4985, "CidrIp": { "Ref": "CidrIpVPC" } }, + { "IpProtocol": "tcp", "FromPort": 8091, "ToPort": 8094, "CidrIp": { "Ref": "CidrIpVPC" } }, + { "IpProtocol": "tcp", "FromPort": 9100, "ToPort": 9105, "CidrIp": { "Ref": "CidrIpVPC" } }, + { "IpProtocol": "tcp", "FromPort": 9998, "ToPort": 9999, "CidrIp": { "Ref": "CidrIpVPC" } }, + { "IpProtocol": "tcp", "FromPort": 11207, "ToPort": 11215, "CidrIp": { "Ref": "CidrIpVPC" } }, + { "IpProtocol": "tcp", "FromPort": 18091, "ToPort": 18093, "CidrIp": { "Ref": "CidrIpVPC" } }, + { "IpProtocol": "tcp", "FromPort": 21100, "ToPort": 21299, "CidrIp": { "Ref": "CidrIpVPC" } } ] } } @@ -194,7 +206,7 @@ def generateSyncGateway(group, rallyAutoScalingGroup): "AvailabilityZones": { "Fn::GetAZs": "" }, "LaunchConfigurationName": { "Ref": groupName + "LaunchConfiguration" }, "MinSize": 0, - "MaxSize": 100, + "MaxSize": 30, "DesiredCapacity": nodeCount } }, @@ -222,8 +234,8 @@ def generateSyncGateway(group, rallyAutoScalingGroup): "stackName=", { "Ref": "AWS::StackName" }, "\n", "baseURL=https://raw.githubusercontent.com/couchbase-partners/amazon-cloud-formation-couchbase/master/scripts/\n", "wget ${baseURL}syncGateway.sh\n", - "chmod +x *.sh\n", - "./syncGateway.sh ${stackName}\n" + "chmod +x *.sh\n" + # "./syncGateway.sh ${stackName}\n" ]] } } @@ -268,10 +280,11 @@ def generateServer(group, rallyAutoScalingGroup): groupName + "AutoScalingGroup": { "Type": "AWS::AutoScaling::AutoScalingGroup", "Properties": { - "AvailabilityZones": { "Fn::GetAZs": "" }, + "VpcId" : { "Ref" : "VPC" }, + "AvailabilityZones": { "Fn::GetAZs": { "Ref" : "AWS::Region" }}, "LaunchConfigurationName": { "Ref": groupName + "LaunchConfiguration" }, "MinSize": 1, - "MaxSize": 100, + "MaxSize": 30, "DesiredCapacity": nodeCount } }, diff --git a/generator/parameters/granular.yaml b/generator/parameters/granular.yaml index 3209e38..7a9a25e 100644 --- a/generator/parameters/granular.yaml +++ b/generator/parameters/granular.yaml @@ -1,5 +1,5 @@ -serverVersion: 5.0.1 -syncGatewayVersion: 1.5.1 +serverVersion: 4.6.3 +#syncGatewayVersion: 1.5.1 cluster: - group: data nodeCount: 2 @@ -8,25 +8,25 @@ cluster: services: - data - group: query - nodeCount: 2 + nodeCount: 1 nodeType: m4.4xlarge dataDiskSize: 100 services: - query - group: index - nodeCount: 2 + nodeCount: 1 nodeType: m4.4xlarge dataDiskSize: 100 services: - index - - group: fts - nodeCount: 2 - nodeType: m4.4xlarge - dataDiskSize: 100 - services: - - fts - - group: syncgateway - nodeCount: 2 - nodeType: m4.4xlarge - services: - - syncGateway + # - group: fts + # nodeCount: 2 + # nodeType: m4.4xlarge + # dataDiskSize: 100 + # services: + # - fts + # - group: syncgateway + # nodeCount: 2 + # nodeType: m4.4xlarge + # services: + # - syncGateway From ef859e8d927fe46ed5729e08660f415228b55101 Mon Sep 17 00:00:00 2001 From: faizan Date: Sun, 25 Feb 2018 12:15:06 +0530 Subject: [PATCH 02/52] Initial working version --- generator/generator.py | 98 +++++++++++++++++++++++------------------- 1 file changed, 53 insertions(+), 45 deletions(-) diff --git a/generator/generator.py b/generator/generator.py index 5eb5242..aa6e802 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -31,18 +31,22 @@ def main(): "Description": "VpcId of your existing Virtual Private Cloud (VPC)", "ConstraintDescription": "must be the VPC Id of an existing Virtual Private Cloud." }, + "Subnets": { + "Type": "List", + "Description": "The list of SubnetIds in your Virtual Private Cloud (VPC)", + "ConstraintDescription": "must be a list of at least two existing subnets associated with at least two different availability zones. They should be residing in the selected Virtual Private Cloud." + }, "CidrIpVPC": { - "Type": "AWS::EC2::VPCCidrBlock", + "Type": "String", "Description": "CIDR Block for existing VPC. Traffic would be allowed from this only", "ConstraintDescription": "must be the VPC Id of an existing Virtual Private Cloud." }, - "Region": { - "Type": "AWS::Region", - "Description": "Region where stack is being deployed", - #"ConstraintDescription": "must be a list of at least two existing subnets associated with at least two different availability zones. They should be residing in the selected Virtual Private Cloud." - "default": "us-west-2" - }, - + "License": { + "Description": "License model can be BYOL or HourlyPricing", + "Type": "String", + "Default": "BYOL", + "ConstraintDescription": "Select BYOL only for GLP as these are pre baked images" + } }, "Mappings": {}, "Resources": {} @@ -64,21 +68,21 @@ def generateMappings(serverVersion): allMappings = { "CouchbaseServer": { "4.6.3": { - "us-east-1": "ami-9e87f2e4", - "us-east-2": "ami-9e87f2e4", - "us-west-1": "ami-9e87f2e4", - "us-west-2": "ami-380dfa40", - "ca-central-1": "ami-492ca92d", - "eu-central-1": "ami-d63ca5b9", - "eu-west-1": "ami-fc99fe85", - "eu-west-2": "ami-cb455faf", - "eu-west-3": "ami-d3dd6bae", - "ap-southeast-1": "ami-17ed956b", - "ap-southeast-2": "ami-c30ef0a1", - "ap-south-1": "ami-c8b0e1a7", - "ap-northeast-1": "ami-75e78a13", - "ap-northeast-2": "ami-888e2de6", - "sa-east-1": "ami-d3571bbf" + "us-east-1": { "BYOL": "ami-a693a3dc", "HourlyPricing": "ami-ef95a595" }, + "us-east-2": { "BYOL": "ami-d97441bc", "HourlyPricing": "ami-62764307" }, + "us-west-1": { "BYOL": "ami-cf8c81af", "HourlyPricing": "ami-c08c81a0" }, + "us-west-2": { "BYOL": "ami-380dfa40", "HourlyPricing": "ami-49a11e31" }, + "ca-central-1": { "BYOL": "ami-9822a7fc", "HourlyPricing": "ami-2e22a74a" }, + "eu-central-1": { "BYOL": "ami-8438a1eb", "HourlyPricing": "ami-9939a0f6" }, + "eu-west-1": { "BYOL": "ami-078aed7e", "HourlyPricing": "ami-7797f00e" }, + "eu-west-2": { "BYOL": "ami-dd455fb9", "HourlyPricing": "ami-be7b61da" }, + "eu-west-3": { "BYOL": "ami-d5dd6ba8", "HourlyPricing": "ami-5bc77126" }, + "ap-southeast-1": { "BYOL": "ami-33ec944f", "HourlyPricing": "ami-13eb936f" }, + "ap-southeast-2": { "BYOL": "ami-8910eeeb", "HourlyPricing": "ami-ec11ef8e" }, + "ap-south-1": { "BYOL": "aami-0d8ddc62", "HourlyPricing": "ami-5db1e032" }, + "ap-northeast-1": { "BYOL": "ami-b0e489d6", "HourlyPricing": "ami-47e48921" }, + "ap-northeast-2": { "BYOL": "ami-ec8d2e82", "HourlyPricing": "ami-e78c2f89" }, + "sa-east-1": { "BYOL": "ami-995519f5", "HourlyPricing": "ami-f5551999" } }, "5.0.1": { "us-east-1": { "BYOL": "ami-a693a3dc", "HourlyPricing": "ami-ef95a595" }, @@ -98,25 +102,25 @@ def generateMappings(serverVersion): "sa-east-1": { "BYOL": "ami-995519f5", "HourlyPricing": "ami-f5551999" } } }, - # "CouchbaseSyncGateway": { - # "1.5.1": { - # "us-east-1": { "BYOL": "ami-8294a4f8", "HourlyPricing": "ami-6d93a317" }, - # "us-east-2": { "BYOL": "ami-0877426d", "HourlyPricing": "ami-0f75406a" }, - # "us-west-1": { "BYOL": "ami-288c8148", "HourlyPricing": "ami-76f2ff16" }, - # "us-west-2": { "BYOL": "ami-589c2320", "HourlyPricing": "ami-41a01f39" }, - # "ca-central-1": { "BYOL": "ami-ad20a5c9", "HourlyPricing": "ami-c22da8a6" }, - # "eu-central-1": { "BYOL": "ami-103aa37f", "HourlyPricing": "ami-d5069fba" }, - # "eu-west-1": { "BYOL": "ami-b696f1cf", "HourlyPricing": "ami-c293f4bb" }, - # "eu-west-2": { "BYOL": "ami-6c445e08", "HourlyPricing": "ami-11445e75" }, - # "eu-west-3": { "BYOL": "ami-c9d86eb4", "HourlyPricing": "ami-58c77125" }, - # "ap-southeast-1": { "BYOL": "ami-10eb936c", "HourlyPricing": "ami-a4ed95d8" }, - # "ap-southeast-2": { "BYOL": "ami-a610eec4", "HourlyPricing": "ami-2911ef4b" }, - # "ap-south-1": { "BYOL": "ami-898cdde6", "HourlyPricing": "ami-058cdd6a" }, - # "ap-northeast-1": { "BYOL": "ami-b9e588df", "HourlyPricing": "ami-b3e588d5" }, - # "ap-northeast-2": { "BYOL": "ami-38933056", "HourlyPricing": "ami-648c2f0a" }, - # "sa-east-1": { "BYOL": "ami-3654185a", "HourlyPricing": "ami-b95a16d5" } - # } - # } + "CouchbaseSyncGateway": { + "1.5.1": { + "us-east-1": { "BYOL": "ami-8294a4f8", "HourlyPricing": "ami-6d93a317" }, + "us-east-2": { "BYOL": "ami-0877426d", "HourlyPricing": "ami-0f75406a" }, + "us-west-1": { "BYOL": "ami-288c8148", "HourlyPricing": "ami-76f2ff16" }, + "us-west-2": { "BYOL": "ami-589c2320", "HourlyPricing": "ami-41a01f39" }, + "ca-central-1": { "BYOL": "ami-ad20a5c9", "HourlyPricing": "ami-c22da8a6" }, + "eu-central-1": { "BYOL": "ami-103aa37f", "HourlyPricing": "ami-d5069fba" }, + "eu-west-1": { "BYOL": "ami-b696f1cf", "HourlyPricing": "ami-c293f4bb" }, + "eu-west-2": { "BYOL": "ami-6c445e08", "HourlyPricing": "ami-11445e75" }, + "eu-west-3": { "BYOL": "ami-c9d86eb4", "HourlyPricing": "ami-58c77125" }, + "ap-southeast-1": { "BYOL": "ami-10eb936c", "HourlyPricing": "ami-a4ed95d8" }, + "ap-southeast-2": { "BYOL": "ami-a610eec4", "HourlyPricing": "ami-2911ef4b" }, + "ap-south-1": { "BYOL": "ami-898cdde6", "HourlyPricing": "ami-058cdd6a" }, + "ap-northeast-1": { "BYOL": "ami-b9e588df", "HourlyPricing": "ami-b3e588d5" }, + "ap-northeast-2": { "BYOL": "ami-38933056", "HourlyPricing": "ami-648c2f0a" }, + "sa-east-1": { "BYOL": "ami-3654185a", "HourlyPricing": "ami-b95a16d5" } + } + } } mappings = { "CouchbaseServer": allMappings["CouchbaseServer"][serverVersion], @@ -172,7 +176,10 @@ def generateMiscResources(): { "IpProtocol": "tcp", "FromPort": 11207, "ToPort": 11215, "CidrIp": { "Ref": "CidrIpVPC" } }, { "IpProtocol": "tcp", "FromPort": 18091, "ToPort": 18093, "CidrIp": { "Ref": "CidrIpVPC" } }, { "IpProtocol": "tcp", "FromPort": 21100, "ToPort": 21299, "CidrIp": { "Ref": "CidrIpVPC" } } - ] + ], + "VpcId": { + "Ref": "VpcId" + } } } } @@ -280,8 +287,9 @@ def generateServer(group, rallyAutoScalingGroup): groupName + "AutoScalingGroup": { "Type": "AWS::AutoScaling::AutoScalingGroup", "Properties": { - "VpcId" : { "Ref" : "VPC" }, - "AvailabilityZones": { "Fn::GetAZs": { "Ref" : "AWS::Region" }}, + # "VpcId" : { "Ref" : "vpcId" }, + # "AvailabilityZones": { "Fn::GetAZs": { "Ref" : "AWS::Region" }}, + "VPCZoneIdentifier": { "Ref": "Subnets" }, "LaunchConfigurationName": { "Ref": groupName + "LaunchConfiguration" }, "MinSize": 1, "MaxSize": 30, From 5b0ddef52a343ba235fb884270f518ef35ccf735 Mon Sep 17 00:00:00 2001 From: faizan Date: Sun, 25 Feb 2018 13:56:15 +0530 Subject: [PATCH 03/52] Setting for private network --- scripts/server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/server.sh b/scripts/server.sh index 2152723..ca26ee9 100755 --- a/scripts/server.sh +++ b/scripts/server.sh @@ -32,7 +32,7 @@ instanceID=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/doc | jq '.instanceId' \ | sed 's/^"\(.*\)"$/\1/' ) -nodePublicDNS=`curl http://169.254.169.254/latest/meta-data/public-hostname` +nodePublicDNS=`curl http://169.254.169.254/latest/meta-data/hostname` echo "Using the settings:" echo adminUsername \'$adminUsername\' From 656a0cbed8c952a766d799e019071964a581e613 Mon Sep 17 00:00:00 2001 From: faizan Date: Sun, 25 Feb 2018 13:58:08 +0530 Subject: [PATCH 04/52] Modified cf templates for private network --- generator/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/generator.py b/generator/generator.py index aa6e802..42926b8 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -270,7 +270,7 @@ def generateServer(group, rallyAutoScalingGroup): "adminPassword=", { "Ref": "Password" }, "\n", "services=" + servicesParameter + "\n", "stackName=", { "Ref": "AWS::StackName" }, "\n", - "baseURL=https://raw.githubusercontent.com/couchbase-partners/amazon-cloud-formation-couchbase/master/scripts/\n", + "baseURL=https://raw.githubusercontent.com/GloballogicPractices/amazon-cloud-formation-couchbase/master/scripts/\n", "wget ${baseURL}server.sh\n", "wget ${baseURL}util.sh\n", "chmod +x *.sh\n", From f2441a6b22a84e738290e698bab291281cbf6d43 Mon Sep 17 00:00:00 2001 From: faizan Date: Sun, 25 Feb 2018 14:32:37 +0530 Subject: [PATCH 05/52] Installing aws-cli --- generator/generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generator/generator.py b/generator/generator.py index 42926b8..eb785c2 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -266,6 +266,8 @@ def generateServer(group, rallyAutoScalingGroup): command = [ "#!/bin/bash\n", "echo 'Running startup script...'\n", + "echo 'Install aws-cli...'\n" + "yum install -y aws-cli" "adminUsername=", { "Ref": "Username" }, "\n", "adminPassword=", { "Ref": "Password" }, "\n", "services=" + servicesParameter + "\n", From 335ce5f1398a353377f6bf8884efaa8cd96631e0 Mon Sep 17 00:00:00 2001 From: faizan Date: Sun, 25 Feb 2018 15:04:45 +0530 Subject: [PATCH 06/52] Assigned a public ip to instances --- generator/generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generator/generator.py b/generator/generator.py index eb785c2..1164173 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -267,7 +267,7 @@ def generateServer(group, rallyAutoScalingGroup): "#!/bin/bash\n", "echo 'Running startup script...'\n", "echo 'Install aws-cli...'\n" - "yum install -y aws-cli" + "yum install -y aws-cli \n" "adminUsername=", { "Ref": "Username" }, "\n", "adminPassword=", { "Ref": "Password" }, "\n", "services=" + servicesParameter + "\n", @@ -303,6 +303,7 @@ def generateServer(group, rallyAutoScalingGroup): "Properties": { "ImageId": { "Fn::FindInMap": [ "CouchbaseServer", { "Ref": "AWS::Region" }, { "Ref": "License" } ] }, "InstanceType": nodeType, + "AssociatePublicIpAddress": True, "SecurityGroups": [ { "Ref": "CouchbaseSecurityGroup" } ], "KeyName": { "Ref": "KeyName" }, "EbsOptimized": True, From 8b4a8aa1501dba0aa2e0ec91aecda118d1175eda Mon Sep 17 00:00:00 2001 From: faizan Date: Sun, 25 Feb 2018 15:54:21 +0530 Subject: [PATCH 07/52] Reverting to public hostname --- generator/generator.py | 2 +- scripts/server.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/generator.py b/generator/generator.py index 1164173..e636034 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -153,7 +153,7 @@ def generateMiscResources(): "Effect": "Allow", "Action": [ "ec2:CreateTags", - "ec2:DescribeInstances", + "ec2:Describe*", "autoscaling:DescribeAutoScalingGroups" ], "Resource": "*" diff --git a/scripts/server.sh b/scripts/server.sh index ca26ee9..2152723 100755 --- a/scripts/server.sh +++ b/scripts/server.sh @@ -32,7 +32,7 @@ instanceID=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/doc | jq '.instanceId' \ | sed 's/^"\(.*\)"$/\1/' ) -nodePublicDNS=`curl http://169.254.169.254/latest/meta-data/hostname` +nodePublicDNS=`curl http://169.254.169.254/latest/meta-data/public-hostname` echo "Using the settings:" echo adminUsername \'$adminUsername\' From c967db971e3ec1c38f68b77bb3645fde3907d7e4 Mon Sep 17 00:00:00 2001 From: faizan Date: Fri, 23 Mar 2018 11:53:40 +0530 Subject: [PATCH 08/52] Added tag to identify couchbase role. This will allow us to add ip addresses to hosted zone in route 53 --- generator/parameters/granular.yaml | 6 +++--- generator/parameters/simple.yaml | 6 +++--- scripts/server.sh | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/generator/parameters/granular.yaml b/generator/parameters/granular.yaml index 7a9a25e..797fdec 100644 --- a/generator/parameters/granular.yaml +++ b/generator/parameters/granular.yaml @@ -1,4 +1,4 @@ -serverVersion: 4.6.3 +serverVersion: 5.0.1 #syncGatewayVersion: 1.5.1 cluster: - group: data @@ -10,11 +10,11 @@ cluster: - group: query nodeCount: 1 nodeType: m4.4xlarge - dataDiskSize: 100 + dataDiskSize: 20 services: - query - group: index - nodeCount: 1 + nodeCount: 2 nodeType: m4.4xlarge dataDiskSize: 100 services: diff --git a/generator/parameters/simple.yaml b/generator/parameters/simple.yaml index 0fb7d78..69c6a06 100644 --- a/generator/parameters/simple.yaml +++ b/generator/parameters/simple.yaml @@ -1,12 +1,12 @@ serverVersion: 5.0.1 -syncGatewayVersion: 1.5.1 +# syncGatewayVersion: 1.5.1 cluster: - group: group1 - nodeCount: 5 + nodeCount: 1 nodeType: m4.4xlarge dataDiskSize: 100 services: - data - query - index - - fts + # - fts diff --git a/scripts/server.sh b/scripts/server.sh index 2152723..cde0a25 100755 --- a/scripts/server.sh +++ b/scripts/server.sh @@ -49,12 +49,12 @@ then aws ec2 create-tags \ --region ${region} \ --resources ${instanceID} \ - --tags Key=Name,Value=${stackName}-ServerRally + --tags Key=Name,Value=${stackName}-ServerRally Key=Role,Value=couchbase else aws ec2 create-tags \ --region ${region} \ --resources ${instanceID} \ - --tags Key=Name,Value=${stackName}-Server + --tags Key=Name,Value=${stackName}-Server Key=Role,Value=couchbase fi cd /opt/couchbase/bin/ From 9f3f492abe4cf1e0828ed1d791dbd278ec466837 Mon Sep 17 00:00:00 2001 From: faizan Date: Thu, 29 Mar 2018 15:49:54 +0530 Subject: [PATCH 09/52] Updated stack to rely on private IP and DNS --- generator/generator.py | 9 ++++++--- scripts/server.sh | 28 ++++++++++++++-------------- scripts/util.sh | 8 ++++---- 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/generator/generator.py b/generator/generator.py index e636034..10be9db 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -33,7 +33,7 @@ def main(): }, "Subnets": { "Type": "List", - "Description": "The list of SubnetIds in your Virtual Private Cloud (VPC)", + "Description": "The list of PRIVATE SubnetIds in your Virtual Private Cloud (VPC)", "ConstraintDescription": "must be a list of at least two existing subnets associated with at least two different availability zones. They should be residing in the selected Virtual Private Cloud." }, "CidrIpVPC": { @@ -88,7 +88,7 @@ def generateMappings(serverVersion): "us-east-1": { "BYOL": "ami-a693a3dc", "HourlyPricing": "ami-ef95a595" }, "us-east-2": { "BYOL": "ami-d97441bc", "HourlyPricing": "ami-62764307" }, "us-west-1": { "BYOL": "ami-cf8c81af", "HourlyPricing": "ami-c08c81a0" }, - "us-west-2": { "BYOL": "ami-269c235e", "HourlyPricing": "ami-49a11e31" }, + "us-west-2": { "BYOL": "ami-603aa218", "HourlyPricing": "ami-49a11e31" }, "ca-central-1": { "BYOL": "ami-9822a7fc", "HourlyPricing": "ami-2e22a74a" }, "eu-central-1": { "BYOL": "ami-8438a1eb", "HourlyPricing": "ami-9939a0f6" }, "eu-west-1": { "BYOL": "ami-078aed7e", "HourlyPricing": "ami-7797f00e" }, @@ -154,6 +154,9 @@ def generateMiscResources(): "Action": [ "ec2:CreateTags", "ec2:Describe*", + "ec2:CreateTags", + "ec2:DescribeInstances", + "cloudwatch:PutMetricData", "autoscaling:DescribeAutoScalingGroups" ], "Resource": "*" @@ -303,7 +306,7 @@ def generateServer(group, rallyAutoScalingGroup): "Properties": { "ImageId": { "Fn::FindInMap": [ "CouchbaseServer", { "Ref": "AWS::Region" }, { "Ref": "License" } ] }, "InstanceType": nodeType, - "AssociatePublicIpAddress": True, + "AssociatePublicIpAddress": False, "SecurityGroups": [ { "Ref": "CouchbaseSecurityGroup" } ], "KeyName": { "Ref": "KeyName" }, "EbsOptimized": True, diff --git a/scripts/server.sh b/scripts/server.sh index cde0a25..7eecab5 100755 --- a/scripts/server.sh +++ b/scripts/server.sh @@ -16,12 +16,12 @@ yum -y install jq if [ -z "$5" ] then echo "This node is part of the autoscaling group that contains the rally point." - rallyPublicDNS=`getRallyPublicDNS` + rallyPrivateDNS=`getrallyPrivateDNS` else rallyAutoScalingGroup=$5 echo "This node is not the rally point and not part of the autoscaling group that contains the rally point." echo rallyAutoScalingGroup \'$rallyAutoScalingGroup\' - rallyPublicDNS=`getRallyPublicDNS ${rallyAutoScalingGroup}` + getrallyPrivateDNS=`getrallyPrivateDNS ${rallyAutoScalingGroup}` fi region=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document \ @@ -32,19 +32,19 @@ instanceID=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/doc | jq '.instanceId' \ | sed 's/^"\(.*\)"$/\1/' ) -nodePublicDNS=`curl http://169.254.169.254/latest/meta-data/public-hostname` +nodePrivateDNS=`curl http://169.254.169.254/latest/meta-data/hostname` echo "Using the settings:" echo adminUsername \'$adminUsername\' echo adminPassword \'$adminPassword\' echo services \'$services\' echo stackName \'$stackName\' -echo rallyPublicDNS \'$rallyPublicDNS\' +echo rallyPrivateDNS \'$rallyPrivateDNS\' echo region \'$region\' echo instanceID \'$instanceID\' -echo nodePublicDNS \'$nodePublicDNS\' +echo nodePrivateDNS \'$nodePrivateDNS\' -if [[ ${rallyPublicDNS} == ${nodePublicDNS} ]] +if [[ ${rallyPrivateDNS} == ${nodePrivateDNS} ]] then aws ec2 create-tags \ --region ${region} \ @@ -64,8 +64,8 @@ output="" while [[ ! $output =~ "SUCCESS" ]] do output=`./couchbase-cli node-init \ - --cluster=$nodePublicDNS \ - --node-init-hostname=$nodePublicDNS \ + --cluster=$nodePrivateDNS \ + --node-init-hostname=$nodePrivateDNS \ --node-init-data-path=/mnt/datadisk/data \ --node-init-index-path=/mnt/datadisk/index \ --user=$adminUsername \ @@ -74,7 +74,7 @@ do sleep 10 done -if [[ $rallyPublicDNS == $nodePublicDNS ]] +if [[ $rallyPrivateDNS == $nodePrivateDNS ]] then totalRAM=$(grep MemTotal /proc/meminfo | awk '{print $2}') dataRAM=$((50 * $totalRAM / 100000)) @@ -82,7 +82,7 @@ then echo "Running couchbase-cli cluster-init" ./couchbase-cli cluster-init \ - --cluster=$nodePublicDNS \ + --cluster=$nodePrivateDNS \ --cluster-username=$adminUsername \ --cluster-password=$adminPassword \ --cluster-ramsize=$dataRAM \ @@ -91,13 +91,13 @@ then else echo "Running couchbase-cli server-add" output="" - while [[ $output != "Server $nodePublicDNS:8091 added" && ! $output =~ "Node is already part of cluster." ]] + while [[ $output != "Server $nodePrivateDNS:8091 added" && ! $output =~ "Node is already part of cluster." ]] do output=`./couchbase-cli server-add \ - --cluster=$rallyPublicDNS \ + --cluster=$rallyPrivateDNS \ --user=$adminUsername \ --pass=$adminPassword \ - --server-add=$nodePublicDNS \ + --server-add=$nodePrivateDNS \ --server-add-username=$adminUsername \ --server-add-password=$adminPassword \ --services=${services}` @@ -110,7 +110,7 @@ else while [[ ! $output =~ "SUCCESS" ]] do output=`./couchbase-cli rebalance \ - --cluster=$rallyPublicDNS \ + --cluster=$rallyPrivateDNS \ --user=$adminUsername \ --pass=$adminPassword` echo rebalance output \'$output\' diff --git a/scripts/util.sh b/scripts/util.sh index bf5c91f..1a91eec 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -21,7 +21,7 @@ formatDataDisk () chgrp couchbase $MOUNTPOINT } -getRallyPublicDNS () +getrallyPrivateDNS () { region=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document \ | jq '.region' \ @@ -62,11 +62,11 @@ getRallyPublicDNS () fi done - rallyPublicDNS=$(aws ec2 describe-instances \ + rallyPrivateDNS=$(aws ec2 describe-instances \ --region ${region} \ - --query 'Reservations[0].Instances[0].NetworkInterfaces[0].Association.PublicDnsName' \ + --query 'Reservations[0].Instances[0].NetworkInterfaces[0].PrivateDnsName' \ --instance-ids ${rallyInstanceID} \ --output text) - echo ${rallyPublicDNS} + echo ${rallyPrivateDNS} } From 94d7e35d6f998ab9d2aba5bf36b02eb7322b9fed Mon Sep 17 00:00:00 2001 From: faizan Date: Thu, 29 Mar 2018 23:57:07 +0530 Subject: [PATCH 10/52] Adding debug to util.sh --- scripts/util.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/util.sh b/scripts/util.sh index 1a91eec..9b35c4a 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -23,6 +23,8 @@ formatDataDisk () getrallyPrivateDNS () { + echo "In get private dns" + region=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document \ | jq '.region' \ | sed 's/^"\(.*\)"$/\1/' ) @@ -68,5 +70,5 @@ getrallyPrivateDNS () --instance-ids ${rallyInstanceID} \ --output text) - echo ${rallyPrivateDNS} + echo rallyPrivateDNS \'${rallyPrivateDNS}\' } From f36ee3995814da20b37b2286463298ba226a239b Mon Sep 17 00:00:00 2001 From: faizan Date: Fri, 30 Mar 2018 00:16:51 +0530 Subject: [PATCH 11/52] Fixed bug with rallyPrivateDNS --- scripts/server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/server.sh b/scripts/server.sh index 7eecab5..46e7b6d 100755 --- a/scripts/server.sh +++ b/scripts/server.sh @@ -21,7 +21,7 @@ else rallyAutoScalingGroup=$5 echo "This node is not the rally point and not part of the autoscaling group that contains the rally point." echo rallyAutoScalingGroup \'$rallyAutoScalingGroup\' - getrallyPrivateDNS=`getrallyPrivateDNS ${rallyAutoScalingGroup}` + rallyPrivateDNS=`getrallyPrivateDNS ${rallyAutoScalingGroup}` fi region=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document \ From f65c3b7f3c4eb3b260f36f03b46f340f4b7c456c Mon Sep 17 00:00:00 2001 From: faizan Date: Fri, 30 Mar 2018 00:24:54 +0530 Subject: [PATCH 12/52] Fixed bug with rallyPrivateDNS --- scripts/util.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/util.sh b/scripts/util.sh index 9b35c4a..4342887 100644 --- a/scripts/util.sh +++ b/scripts/util.sh @@ -23,8 +23,7 @@ formatDataDisk () getrallyPrivateDNS () { - echo "In get private dns" - + region=$(curl -s http://169.254.169.254/latest/dynamic/instance-identity/document \ | jq '.region' \ | sed 's/^"\(.*\)"$/\1/' ) @@ -70,5 +69,5 @@ getrallyPrivateDNS () --instance-ids ${rallyInstanceID} \ --output text) - echo rallyPrivateDNS \'${rallyPrivateDNS}\' + echo ${rallyPrivateDNS} } From 83f6d370fb97e6606b83ed4263de4e2c98619204 Mon Sep 17 00:00:00 2001 From: faizan Date: Fri, 30 Mar 2018 01:56:03 +0530 Subject: [PATCH 13/52] Added cloudwatch roles and updated to AMZN2 AMI for us-west-2 --- generator/generator.py | 13 +++++++++---- generator/parameters/granular.yaml | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/generator/generator.py b/generator/generator.py index e636034..5b67be1 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -88,7 +88,7 @@ def generateMappings(serverVersion): "us-east-1": { "BYOL": "ami-a693a3dc", "HourlyPricing": "ami-ef95a595" }, "us-east-2": { "BYOL": "ami-d97441bc", "HourlyPricing": "ami-62764307" }, "us-west-1": { "BYOL": "ami-cf8c81af", "HourlyPricing": "ami-c08c81a0" }, - "us-west-2": { "BYOL": "ami-269c235e", "HourlyPricing": "ami-49a11e31" }, + "us-west-2": { "BYOL": "ami-603aa218", "HourlyPricing": "ami-49a11e31" }, "ca-central-1": { "BYOL": "ami-9822a7fc", "HourlyPricing": "ami-2e22a74a" }, "eu-central-1": { "BYOL": "ami-8438a1eb", "HourlyPricing": "ami-9939a0f6" }, "eu-west-1": { "BYOL": "ami-078aed7e", "HourlyPricing": "ami-7797f00e" }, @@ -152,9 +152,14 @@ def generateMiscResources(): "Statement": [{ "Effect": "Allow", "Action": [ - "ec2:CreateTags", - "ec2:Describe*", - "autoscaling:DescribeAutoScalingGroups" + "ec2:CreateTags", + "ec2:Describe*", + "autoscaling:DescribeAutoScalingGroups", + "cloudwatch:PutMetricData", + "cloudwatch:GetMetricStatistics", + "cloudwatch:ListMetrics", + "ec2:DescribeTags", + "autoscaling:DescribeAutoScalingGroups" ], "Resource": "*" }] diff --git a/generator/parameters/granular.yaml b/generator/parameters/granular.yaml index 797fdec..68918ca 100644 --- a/generator/parameters/granular.yaml +++ b/generator/parameters/granular.yaml @@ -10,7 +10,7 @@ cluster: - group: query nodeCount: 1 nodeType: m4.4xlarge - dataDiskSize: 20 + dataDiskSize: 5 services: - query - group: index From 63bf7f5744467460af9b3afbe9f07622b4a80af7 Mon Sep 17 00:00:00 2001 From: faizan Date: Sun, 1 Apr 2018 13:59:29 +0530 Subject: [PATCH 14/52] Updated to prod AMI and increased EBS vloumes to 200G --- generator/generator.py | 2 +- generator/parameters/granular.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/generator/generator.py b/generator/generator.py index 5b67be1..250c660 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -88,7 +88,7 @@ def generateMappings(serverVersion): "us-east-1": { "BYOL": "ami-a693a3dc", "HourlyPricing": "ami-ef95a595" }, "us-east-2": { "BYOL": "ami-d97441bc", "HourlyPricing": "ami-62764307" }, "us-west-1": { "BYOL": "ami-cf8c81af", "HourlyPricing": "ami-c08c81a0" }, - "us-west-2": { "BYOL": "ami-603aa218", "HourlyPricing": "ami-49a11e31" }, + "us-west-2": { "BYOL": "ami-4210753a", "HourlyPricing": "ami-49a11e31" }, "ca-central-1": { "BYOL": "ami-9822a7fc", "HourlyPricing": "ami-2e22a74a" }, "eu-central-1": { "BYOL": "ami-8438a1eb", "HourlyPricing": "ami-9939a0f6" }, "eu-west-1": { "BYOL": "ami-078aed7e", "HourlyPricing": "ami-7797f00e" }, diff --git a/generator/parameters/granular.yaml b/generator/parameters/granular.yaml index 68918ca..91202c0 100644 --- a/generator/parameters/granular.yaml +++ b/generator/parameters/granular.yaml @@ -4,7 +4,7 @@ cluster: - group: data nodeCount: 2 nodeType: m4.4xlarge - dataDiskSize: 100 + dataDiskSize: 200 services: - data - group: query @@ -16,7 +16,7 @@ cluster: - group: index nodeCount: 2 nodeType: m4.4xlarge - dataDiskSize: 100 + dataDiskSize: 200 services: - index # - group: fts From 282a04a3d8137d6c0cfc8f8f3ac9073ea381f3f7 Mon Sep 17 00:00:00 2001 From: faizan Date: Sun, 1 Apr 2018 14:05:17 +0530 Subject: [PATCH 15/52] Update simple stack with prod AMI for us-west-2 and increased disk size to 200 --- generator/parameters/simple.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/parameters/simple.yaml b/generator/parameters/simple.yaml index 69c6a06..b17b701 100644 --- a/generator/parameters/simple.yaml +++ b/generator/parameters/simple.yaml @@ -4,7 +4,7 @@ cluster: - group: group1 nodeCount: 1 nodeType: m4.4xlarge - dataDiskSize: 100 + dataDiskSize: 200 services: - data - query From 71bb4680ac2ab7f1ed0cef92fb6e45e990d625ef Mon Sep 17 00:00:00 2001 From: faizan Date: Mon, 2 Apr 2018 13:33:42 +0530 Subject: [PATCH 16/52] Updated AMIs to couchbase provided AMIs as we do not have a requirement for ECS agent any more --- generator/generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/generator.py b/generator/generator.py index 250c660..93e1dac 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -88,13 +88,13 @@ def generateMappings(serverVersion): "us-east-1": { "BYOL": "ami-a693a3dc", "HourlyPricing": "ami-ef95a595" }, "us-east-2": { "BYOL": "ami-d97441bc", "HourlyPricing": "ami-62764307" }, "us-west-1": { "BYOL": "ami-cf8c81af", "HourlyPricing": "ami-c08c81a0" }, - "us-west-2": { "BYOL": "ami-4210753a", "HourlyPricing": "ami-49a11e31" }, + "us-west-2": { "BYOL": "ami-269c235e", "HourlyPricing": "ami-49a11e31" }, "ca-central-1": { "BYOL": "ami-9822a7fc", "HourlyPricing": "ami-2e22a74a" }, "eu-central-1": { "BYOL": "ami-8438a1eb", "HourlyPricing": "ami-9939a0f6" }, "eu-west-1": { "BYOL": "ami-078aed7e", "HourlyPricing": "ami-7797f00e" }, "eu-west-2": { "BYOL": "ami-dd455fb9", "HourlyPricing": "ami-be7b61da" }, "eu-west-3": { "BYOL": "ami-d5dd6ba8", "HourlyPricing": "ami-5bc77126" }, - "ap-southeast-1": { "BYOL": "ami-33ec944f", "HourlyPricing": "ami-13eb936f" }, + "ap-southeast-1": { "BYOL": "aami-33ec944f", "HourlyPricing": "ami-13eb936f" }, "ap-southeast-2": { "BYOL": "ami-8910eeeb", "HourlyPricing": "ami-ec11ef8e" }, "ap-south-1": { "BYOL": "aami-0d8ddc62", "HourlyPricing": "ami-5db1e032" }, "ap-northeast-1": { "BYOL": "ami-b0e489d6", "HourlyPricing": "ami-47e48921" }, From 94d839513244c954a08990f5724f6dea3561c2a4 Mon Sep 17 00:00:00 2001 From: faizan Date: Mon, 2 Apr 2018 13:34:57 +0530 Subject: [PATCH 17/52] Fixed bug with AMI name for singapore region --- generator/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/generator.py b/generator/generator.py index 93e1dac..774e008 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -94,7 +94,7 @@ def generateMappings(serverVersion): "eu-west-1": { "BYOL": "ami-078aed7e", "HourlyPricing": "ami-7797f00e" }, "eu-west-2": { "BYOL": "ami-dd455fb9", "HourlyPricing": "ami-be7b61da" }, "eu-west-3": { "BYOL": "ami-d5dd6ba8", "HourlyPricing": "ami-5bc77126" }, - "ap-southeast-1": { "BYOL": "aami-33ec944f", "HourlyPricing": "ami-13eb936f" }, + "ap-southeast-1": { "BYOL": "ami-33ec944f", "HourlyPricing": "ami-13eb936f" }, "ap-southeast-2": { "BYOL": "ami-8910eeeb", "HourlyPricing": "ami-ec11ef8e" }, "ap-south-1": { "BYOL": "aami-0d8ddc62", "HourlyPricing": "ami-5db1e032" }, "ap-northeast-1": { "BYOL": "ami-b0e489d6", "HourlyPricing": "ami-47e48921" }, From 69078af8ef776a4cbe9590d75d164384101ab480 Mon Sep 17 00:00:00 2001 From: faizan Date: Mon, 2 Apr 2018 14:04:39 +0530 Subject: [PATCH 18/52] Tested CF templates with private IPs. Working fine --- generator/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/generator.py b/generator/generator.py index 10be9db..4470685 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -88,7 +88,7 @@ def generateMappings(serverVersion): "us-east-1": { "BYOL": "ami-a693a3dc", "HourlyPricing": "ami-ef95a595" }, "us-east-2": { "BYOL": "ami-d97441bc", "HourlyPricing": "ami-62764307" }, "us-west-1": { "BYOL": "ami-cf8c81af", "HourlyPricing": "ami-c08c81a0" }, - "us-west-2": { "BYOL": "ami-603aa218", "HourlyPricing": "ami-49a11e31" }, + "us-west-2": { "BYOL": "ami-269c235e", "HourlyPricing": "ami-49a11e31" }, "ca-central-1": { "BYOL": "ami-9822a7fc", "HourlyPricing": "ami-2e22a74a" }, "eu-central-1": { "BYOL": "ami-8438a1eb", "HourlyPricing": "ami-9939a0f6" }, "eu-west-1": { "BYOL": "ami-078aed7e", "HourlyPricing": "ami-7797f00e" }, From ec786a612106852af71bdee1d077f6599699cd02 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Mon, 16 Apr 2018 12:07:04 +0530 Subject: [PATCH 19/52] Update generator.py Added IAM Cloudwatch Policies - PutDashboard PutMetricAlarm --- generator/generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generator/generator.py b/generator/generator.py index 2f76614..38a5daa 100644 --- a/generator/generator.py +++ b/generator/generator.py @@ -156,6 +156,8 @@ def generateMiscResources(): "ec2:Describe*", "autoscaling:DescribeAutoScalingGroups", "cloudwatch:PutMetricData", + "cloudwatch:PutDashboard", + "cloudwatch:PutMetricAlarm", "cloudwatch:GetMetricStatistics", "cloudwatch:ListMetrics", "ec2:DescribeTags", From 5d7540b2172ebfd1bcc2a1940efe1dec3d96f334 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Thu, 26 Apr 2018 16:53:37 +0530 Subject: [PATCH 20/52] cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 113 +++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 scripts/cloudwatch-alarms.sh diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh new file mode 100644 index 0000000..de7fe77 --- /dev/null +++ b/scripts/cloudwatch-alarms.sh @@ -0,0 +1,113 @@ +#!/bin/sh +#=========================================================================================================================== +# +# FILE: create-cloudwatch-alarms.sh +# Pre-requisite - Disk and Mem Utlization Custom metrices should be implemented first in order to create Alarm on them. +# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/mon-scripts.html +# DESCRIPTION: Script creates 4 alarms for instance with PRIVATE_IP +# It does follow steps: +# 1. Creates high cpu usage alarm +# 2. Creates instance statusCheck alarm +# 3. Create Alarm to check disk utilization +# 4. Create Alarm on Memory utilization +#============================================================================================================================ +# Get instance id and name tag +PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) +INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) +INSTANCE_NAME=$(ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') + +echo "${PRIVATE_IP}" | grep -q -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" +if [ $? -ne 0 ] +then + echo "### Usage: $0 " + echo "### IP address have to be presented in format X.X.X.X" + exit 1 +fi + +# Trying auto-detect AWS region +AWS_DEFAULT_REGION=$(curl -s --connect-timeout 2 http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e 's/\([1-9]\).$/\1/g') + +# Export environment variables for awscli +export AWS_DEFAULT_REGION +export AWS_DEFAULT_OUTPUT="text" + +if [ "${INSTANCE_NAME}" = "None" ] +then + echo "### Could not find instance with IP ${PRIVATE_IP}" + exit 1 +fi + +# 1) Create high CPU usage metric +ARN_OF_SNS_TOPIC="arn:aws:sns:us-west-2:953030164212:SNS" +CPU_USAGE=70 + +aws cloudwatch put-metric-alarm \ + --alarm-name "${INSTANCE_NAME}-cpu"\ + --alarm-description "Alarm when CPU exceeds ${CPU_USAGE}%"\ + --actions-enabled\ + --ok-actions "${ARN_OF_SNS_TOPIC}"\ + --alarm-actions "${ARN_OF_SNS_TOPIC}"\ + --insufficient-data-actions "${ARN_OF_SNS_TOPIC}"\ + --metric-name CPUUtilization\ + --namespace AWS/EC2\ + --statistic Average\ + --dimensions Name=InstanceId,Value=${INSTANCE_ID}\ + --period 60\ + --threshold ${CPU_USAGE}\ + --comparison-operator GreaterThanThreshold\ + --evaluation-periods 1\ + --unit Percent + +# 2) Create status check metric +aws cloudwatch put-metric-alarm \ + --alarm-name "${INSTANCE_NAME}-status"\ + --alarm-description "Alarm when statusCheck failed"\ + --actions-enabled\ + --ok-actions "${ARN_OF_SNS_TOPIC}"\ + --alarm-actions "${ARN_OF_SNS_TOPIC}"\ + --insufficient-data-actions "${ARN_OF_SNS_TOPIC}"\ + --metric-name StatusCheckFailed\ + --namespace AWS/EC2\ + --statistic Maximum\ + --dimensions Name=InstanceId,Value=${INSTANCE_ID}\ + --period 60\ + --threshold 1\ + --comparison-operator GreaterThanOrEqualToThreshold\ + --evaluation-periods 1\ + --unit Count +# 3) Create Alarm to check disk utilization +aws cloudwatch put-metric-alarm \ + --alarm-name "${INSTANCE_NAME}-Disk-Utl"\ + --alarm-description "Alarm when Disk usage exceed 85 percent"\ + --actions-enabled \ + --ok-actions "${ARN_OF_SNS_TOPIC}"\ + --alarm-actions "${ARN_OF_SNS_TOPIC}"\ + --insufficient-data-actions "${ARN_OF_SNS_TOPIC}"\ + --metric-name DiskSpaceUtilization\ + --namespace System/Linux \ + --statistic Maximum\ + --dimensions Name=InstanceId,Value=${INSTANCE_ID}\ + --period 60\ + --threshold 1\ + --comparison-operator GreaterThanOrEqualToThreshold\ + --evaluation-periods 1\ + --unit Percent +# 4) Creat Alarm on Memory utilization + + +aws cloudwatch put-metric-alarm \ + --alarm-name "${INSTANCE_NAME}-Mem-Utl"\ + --alarm-description "Alarm when Memory usage exceed 80 percent"\ + --actions-enabled \ + --ok-actions "${ARN_OF_SNS_TOPIC}"\ + --alarm-actions "${ARN_OF_SNS_TOPIC}"\ + --insufficient-data-actions "${ARN_OF_SNS_TOPIC}"\ + --metric-name MemoryUtilization\ + --namespace System/Linux \ + --statistic Maximum\ + --dimensions Name=InstanceId,Value=${INSTANCE_ID}\ + --period 60\ + --threshold 1\ + --comparison-operator GreaterThanOrEqualToThreshold\ + --evaluation-periods 1\ + --unit Percent From 4b4a35db76fde7bb62c82b3572dfa4e36d4d9aec Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Fri, 27 Apr 2018 17:48:16 +0530 Subject: [PATCH 21/52] Update cloudwatch-alarms.sh Added env variable --- scripts/cloudwatch-alarms.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index de7fe77..9a7a8cf 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -12,6 +12,12 @@ # 4. Create Alarm on Memory utilization #============================================================================================================================ # Get instance id and name tag + +echo "Running cloudwatch-alarms.sh" +envVar=$1 + +if [ "${envVar}" = "Pre-prod" ] + then PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) INSTANCE_NAME=$(ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') @@ -23,7 +29,7 @@ then echo "### IP address have to be presented in format X.X.X.X" exit 1 fi - +fi # Trying auto-detect AWS region AWS_DEFAULT_REGION=$(curl -s --connect-timeout 2 http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e 's/\([1-9]\).$/\1/g') From dabd36760aa59bef46efd39eebe484b63e4d7cb9 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Sun, 29 Apr 2018 10:27:56 +0530 Subject: [PATCH 22/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 9a7a8cf..d664878 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -1,26 +1,27 @@ #!/bin/sh -#=========================================================================================================================== +#=================================================================================== # # FILE: create-cloudwatch-alarms.sh -# Pre-requisite - Disk and Mem Utlization Custom metrices should be implemented first in order to create Alarm on them. -# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/mon-scripts.html -# DESCRIPTION: Script creates 4 alarms for instance with PRIVATE_IP +# +# USAGE: create-cloudwatch-alarms.sh +# +# DESCRIPTION: Script creates 2 alarms for instance with PRIVATE_IP # It does follow steps: # 1. Creates high cpu usage alarm # 2. Creates instance statusCheck alarm -# 3. Create Alarm to check disk utilization -# 4. Create Alarm on Memory utilization -#============================================================================================================================ +# +#=================================================================================== # Get instance id and name tag - echo "Running cloudwatch-alarms.sh" + envVar=$1 if [ "${envVar}" = "Pre-prod" ] - then +then + PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) -INSTANCE_NAME=$(ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') +INSTANCE_NAME=$(ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') echo "${PRIVATE_IP}" | grep -q -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" if [ $? -ne 0 ] @@ -29,12 +30,12 @@ then echo "### IP address have to be presented in format X.X.X.X" exit 1 fi -fi + # Trying auto-detect AWS region AWS_DEFAULT_REGION=$(curl -s --connect-timeout 2 http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e 's/\([1-9]\).$/\1/g') # Export environment variables for awscli -export AWS_DEFAULT_REGION +export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-ap-southeast-1}" export AWS_DEFAULT_OUTPUT="text" if [ "${INSTANCE_NAME}" = "None" ] @@ -42,8 +43,8 @@ then echo "### Could not find instance with IP ${PRIVATE_IP}" exit 1 fi - -# 1) Create high CPU usage metric +fi +# 2) Create high CPU usage metric ARN_OF_SNS_TOPIC="arn:aws:sns:us-west-2:953030164212:SNS" CPU_USAGE=70 @@ -64,7 +65,7 @@ aws cloudwatch put-metric-alarm \ --evaluation-periods 1\ --unit Percent -# 2) Create status check metric +# 3) Create status check metric aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-status"\ --alarm-description "Alarm when statusCheck failed"\ From a8d1384c037d33aa1464e4fd299f8cee824b1e6f Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Sun, 29 Apr 2018 11:42:42 +0530 Subject: [PATCH 23/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index d664878..19fc8f0 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -1,29 +1,31 @@ #!/bin/sh -#=================================================================================== +#=========================================================================================================================== # # FILE: create-cloudwatch-alarms.sh -# -# USAGE: create-cloudwatch-alarms.sh -# -# DESCRIPTION: Script creates 2 alarms for instance with PRIVATE_IP +# Pre-requisite - Disk and Mem Utlization Custom metrices should be implemented first in order to create Alarm on them. +# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/mon-scripts.html +# DESCRIPTION: Script creates 4 alarms for instance with PRIVATE_IP # It does follow steps: # 1. Creates high cpu usage alarm # 2. Creates instance statusCheck alarm -# -#=================================================================================== -# Get instance id and name tag -echo "Running cloudwatch-alarms.sh" +# 3. Create Alarm to check disk utilization +# 4. Create Alarm on Memory utilization +#============================================================================================================================ +echo "Running cloudwatch-alarms.sh" envVar=$1 if [ "${envVar}" = "Pre-prod" ] then +# Get instance id and name tag + PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) -INSTANCE_NAME=$(ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') +INSTANCE_NAME=$(ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') echo "${PRIVATE_IP}" | grep -q -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" + if [ $? -ne 0 ] then echo "### Usage: $0 " @@ -35,7 +37,7 @@ fi AWS_DEFAULT_REGION=$(curl -s --connect-timeout 2 http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e 's/\([1-9]\).$/\1/g') # Export environment variables for awscli -export AWS_DEFAULT_REGION="${AWS_DEFAULT_REGION:-ap-southeast-1}" +export AWS_DEFAULT_REGION export AWS_DEFAULT_OUTPUT="text" if [ "${INSTANCE_NAME}" = "None" ] @@ -44,7 +46,7 @@ then exit 1 fi fi -# 2) Create high CPU usage metric +# 1) Create high CPU usage metric ARN_OF_SNS_TOPIC="arn:aws:sns:us-west-2:953030164212:SNS" CPU_USAGE=70 @@ -65,7 +67,7 @@ aws cloudwatch put-metric-alarm \ --evaluation-periods 1\ --unit Percent -# 3) Create status check metric +# 2) Create status check metric aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-status"\ --alarm-description "Alarm when statusCheck failed"\ @@ -82,6 +84,7 @@ aws cloudwatch put-metric-alarm \ --comparison-operator GreaterThanOrEqualToThreshold\ --evaluation-periods 1\ --unit Count + # 3) Create Alarm to check disk utilization aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-Disk-Utl"\ From 1f1ca1723ff1ad5ca0fe9c1a9c2628cef0869834 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Sun, 29 Apr 2018 13:32:27 +0530 Subject: [PATCH 24/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 19fc8f0..32d04a1 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -46,6 +46,10 @@ then exit 1 fi fi + +echo "Using variable:" +echo envVar \'$envVar\' + # 1) Create high CPU usage metric ARN_OF_SNS_TOPIC="arn:aws:sns:us-west-2:953030164212:SNS" CPU_USAGE=70 @@ -104,7 +108,6 @@ aws cloudwatch put-metric-alarm \ --unit Percent # 4) Creat Alarm on Memory utilization - aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-Mem-Utl"\ --alarm-description "Alarm when Memory usage exceed 80 percent"\ From 2b2153b9211c347c8af026d5317dfad1f630e806 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Sun, 29 Apr 2018 14:07:55 +0530 Subject: [PATCH 25/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 32d04a1..e65358e 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -46,15 +46,13 @@ then exit 1 fi fi - -echo "Using variable:" +echo "Using the settings:" echo envVar \'$envVar\' - # 1) Create high CPU usage metric ARN_OF_SNS_TOPIC="arn:aws:sns:us-west-2:953030164212:SNS" CPU_USAGE=70 -aws cloudwatch put-metric-alarm \ +aws cloudwatch put-metric-alarm ${DRYRUN}\ --alarm-name "${INSTANCE_NAME}-cpu"\ --alarm-description "Alarm when CPU exceeds ${CPU_USAGE}%"\ --actions-enabled\ @@ -72,7 +70,7 @@ aws cloudwatch put-metric-alarm \ --unit Percent # 2) Create status check metric -aws cloudwatch put-metric-alarm \ +aws cloudwatch put-metric-alarm ${DRYRUN}\ --alarm-name "${INSTANCE_NAME}-status"\ --alarm-description "Alarm when statusCheck failed"\ --actions-enabled\ @@ -88,9 +86,9 @@ aws cloudwatch put-metric-alarm \ --comparison-operator GreaterThanOrEqualToThreshold\ --evaluation-periods 1\ --unit Count - + # 3) Create Alarm to check disk utilization -aws cloudwatch put-metric-alarm \ +aws cloudwatch put-metric-alarm ${DRYRUN}\ --alarm-name "${INSTANCE_NAME}-Disk-Utl"\ --alarm-description "Alarm when Disk usage exceed 85 percent"\ --actions-enabled \ @@ -108,7 +106,8 @@ aws cloudwatch put-metric-alarm \ --unit Percent # 4) Creat Alarm on Memory utilization -aws cloudwatch put-metric-alarm \ + +aws cloudwatch put-metric-alarm ${DRYRUN} \ --alarm-name "${INSTANCE_NAME}-Mem-Utl"\ --alarm-description "Alarm when Memory usage exceed 80 percent"\ --actions-enabled \ From 0dd53b7934d4a6752a6e7ff1d019eaf2f248c782 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Sun, 29 Apr 2018 14:40:56 +0530 Subject: [PATCH 26/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index e65358e..32d04a1 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -46,13 +46,15 @@ then exit 1 fi fi -echo "Using the settings:" + +echo "Using variable:" echo envVar \'$envVar\' + # 1) Create high CPU usage metric ARN_OF_SNS_TOPIC="arn:aws:sns:us-west-2:953030164212:SNS" CPU_USAGE=70 -aws cloudwatch put-metric-alarm ${DRYRUN}\ +aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-cpu"\ --alarm-description "Alarm when CPU exceeds ${CPU_USAGE}%"\ --actions-enabled\ @@ -70,7 +72,7 @@ aws cloudwatch put-metric-alarm ${DRYRUN}\ --unit Percent # 2) Create status check metric -aws cloudwatch put-metric-alarm ${DRYRUN}\ +aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-status"\ --alarm-description "Alarm when statusCheck failed"\ --actions-enabled\ @@ -86,9 +88,9 @@ aws cloudwatch put-metric-alarm ${DRYRUN}\ --comparison-operator GreaterThanOrEqualToThreshold\ --evaluation-periods 1\ --unit Count - + # 3) Create Alarm to check disk utilization -aws cloudwatch put-metric-alarm ${DRYRUN}\ +aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-Disk-Utl"\ --alarm-description "Alarm when Disk usage exceed 85 percent"\ --actions-enabled \ @@ -106,8 +108,7 @@ aws cloudwatch put-metric-alarm ${DRYRUN}\ --unit Percent # 4) Creat Alarm on Memory utilization - -aws cloudwatch put-metric-alarm ${DRYRUN} \ +aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-Mem-Utl"\ --alarm-description "Alarm when Memory usage exceed 80 percent"\ --actions-enabled \ From 5243028ebcfe9718aad0a6efaad86201929cddb5 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Sun, 29 Apr 2018 15:11:59 +0530 Subject: [PATCH 27/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 32d04a1..3ac07d8 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -40,21 +40,19 @@ AWS_DEFAULT_REGION=$(curl -s --connect-timeout 2 http://169.254.169.254/latest/m export AWS_DEFAULT_REGION export AWS_DEFAULT_OUTPUT="text" -if [ "${INSTANCE_NAME}" = "None" ] -then - echo "### Could not find instance with IP ${PRIVATE_IP}" - exit 1 -fi -fi +#if [ "${INSTANCE_NAME}" = "None" ] +#then +# echo "### Could not find instance with IP ${PRIVATE_IP}" +# exit 1 +#fi -echo "Using variable:" +echo "Using the settings:" echo envVar \'$envVar\' - # 1) Create high CPU usage metric ARN_OF_SNS_TOPIC="arn:aws:sns:us-west-2:953030164212:SNS" CPU_USAGE=70 -aws cloudwatch put-metric-alarm \ +aws cloudwatch put-metric-alarm ${DRYRUN}\ --alarm-name "${INSTANCE_NAME}-cpu"\ --alarm-description "Alarm when CPU exceeds ${CPU_USAGE}%"\ --actions-enabled\ @@ -72,7 +70,7 @@ aws cloudwatch put-metric-alarm \ --unit Percent # 2) Create status check metric -aws cloudwatch put-metric-alarm \ +aws cloudwatch put-metric-alarm ${DRYRUN}\ --alarm-name "${INSTANCE_NAME}-status"\ --alarm-description "Alarm when statusCheck failed"\ --actions-enabled\ @@ -88,9 +86,9 @@ aws cloudwatch put-metric-alarm \ --comparison-operator GreaterThanOrEqualToThreshold\ --evaluation-periods 1\ --unit Count - + # 3) Create Alarm to check disk utilization -aws cloudwatch put-metric-alarm \ +aws cloudwatch put-metric-alarm ${DRYRUN}\ --alarm-name "${INSTANCE_NAME}-Disk-Utl"\ --alarm-description "Alarm when Disk usage exceed 85 percent"\ --actions-enabled \ @@ -108,7 +106,8 @@ aws cloudwatch put-metric-alarm \ --unit Percent # 4) Creat Alarm on Memory utilization -aws cloudwatch put-metric-alarm \ + +aws cloudwatch put-metric-alarm ${DRYRUN} \ --alarm-name "${INSTANCE_NAME}-Mem-Utl"\ --alarm-description "Alarm when Memory usage exceed 80 percent"\ --actions-enabled \ @@ -124,3 +123,6 @@ aws cloudwatch put-metric-alarm \ --comparison-operator GreaterThanOrEqualToThreshold\ --evaluation-periods 1\ --unit Percent +else +echo "$envVar does not require CloudWatch Alarms" +fi From 6851a9c39944a8f5075dbbee77fbe2407454a445 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Sun, 29 Apr 2018 20:14:09 +0530 Subject: [PATCH 28/52] Update cloudwatch-alarms.sh sleep for 20 sec for tag creation --- scripts/cloudwatch-alarms.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 3ac07d8..4c2c0dd 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -17,7 +17,8 @@ envVar=$1 if [ "${envVar}" = "Pre-prod" ] then - +echo "waiting for tag completion" +sleep 20 # Get instance id and name tag PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) From 1e5df76930bbe6383556b5a39bda3126cc8598a4 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Sun, 29 Apr 2018 20:35:06 +0530 Subject: [PATCH 29/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 4c2c0dd..a30695b 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -17,8 +17,8 @@ envVar=$1 if [ "${envVar}" = "Pre-prod" ] then -echo "waiting for tag completion" -sleep 20 +echo "waiting for tag completion..." +sleep 60 # Get instance id and name tag PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) From 22f43a1ef82b99906ab512e97fc28f59ae73c155 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Sun, 29 Apr 2018 21:06:57 +0530 Subject: [PATCH 30/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index a30695b..086f1e9 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -47,13 +47,13 @@ export AWS_DEFAULT_OUTPUT="text" # exit 1 #fi -echo "Using the settings:" +echo "Using variables:" echo envVar \'$envVar\' # 1) Create high CPU usage metric ARN_OF_SNS_TOPIC="arn:aws:sns:us-west-2:953030164212:SNS" CPU_USAGE=70 -aws cloudwatch put-metric-alarm ${DRYRUN}\ +aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-cpu"\ --alarm-description "Alarm when CPU exceeds ${CPU_USAGE}%"\ --actions-enabled\ @@ -71,7 +71,7 @@ aws cloudwatch put-metric-alarm ${DRYRUN}\ --unit Percent # 2) Create status check metric -aws cloudwatch put-metric-alarm ${DRYRUN}\ +aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-status"\ --alarm-description "Alarm when statusCheck failed"\ --actions-enabled\ @@ -89,7 +89,7 @@ aws cloudwatch put-metric-alarm ${DRYRUN}\ --unit Count # 3) Create Alarm to check disk utilization -aws cloudwatch put-metric-alarm ${DRYRUN}\ +aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-Disk-Utl"\ --alarm-description "Alarm when Disk usage exceed 85 percent"\ --actions-enabled \ @@ -108,7 +108,7 @@ aws cloudwatch put-metric-alarm ${DRYRUN}\ # 4) Creat Alarm on Memory utilization -aws cloudwatch put-metric-alarm ${DRYRUN} \ +aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_NAME}-Mem-Utl"\ --alarm-description "Alarm when Memory usage exceed 80 percent"\ --actions-enabled \ @@ -124,6 +124,8 @@ aws cloudwatch put-metric-alarm ${DRYRUN} \ --comparison-operator GreaterThanOrEqualToThreshold\ --evaluation-periods 1\ --unit Percent + +echo "CloudWatch AlarmSetup completed" else echo "$envVar does not require CloudWatch Alarms" fi From 780ed54f92636e39f0cf0d33647aa23d8740effc Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Sun, 29 Apr 2018 21:09:30 +0530 Subject: [PATCH 31/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 086f1e9..0fd5f46 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -29,7 +29,7 @@ echo "${PRIVATE_IP}" | grep -q -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1, if [ $? -ne 0 ] then - echo "### Usage: $0 " + echo "### Usage: $0 $PRIVATE_IP" echo "### IP address have to be presented in format X.X.X.X" exit 1 fi From e471cecb0c1d3813cf20c2d551964836a97c1aec Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Mon, 30 Apr 2018 17:24:06 +0530 Subject: [PATCH 32/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 0fd5f46..f49c115 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -18,7 +18,7 @@ envVar=$1 if [ "${envVar}" = "Pre-prod" ] then echo "waiting for tag completion..." -sleep 60 +sleep 120 # Get instance id and name tag PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) From d3ef4acd79db08bb1d13371e2e28b1a05b7c8e36 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Mon, 30 Apr 2018 17:38:45 +0530 Subject: [PATCH 33/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index f49c115..77e6863 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -18,7 +18,7 @@ envVar=$1 if [ "${envVar}" = "Pre-prod" ] then echo "waiting for tag completion..." -sleep 120 +sleep 10 # Get instance id and name tag PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) From ad9f28d8e1bd8b47ad8830e2c0eeadf509a8283a Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Mon, 30 Apr 2018 17:57:55 +0530 Subject: [PATCH 34/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 77e6863..f66e06b 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -20,7 +20,7 @@ then echo "waiting for tag completion..." sleep 10 # Get instance id and name tag - +cd /usr/bin/aws PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) INSTANCE_NAME=$(ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') From 01bd85c3d8371771512fe8a792620efbe4182929 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Mon, 30 Apr 2018 17:59:06 +0530 Subject: [PATCH 35/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index f66e06b..c9bfb0d 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -21,6 +21,7 @@ echo "waiting for tag completion..." sleep 10 # Get instance id and name tag cd /usr/bin/aws +export HOME=/root PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) INSTANCE_NAME=$(ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') From 6caa47776dbec986a3dc338dbd44d371e7f152b9 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Mon, 30 Apr 2018 18:00:12 +0530 Subject: [PATCH 36/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index c9bfb0d..4544385 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -22,6 +22,7 @@ sleep 10 # Get instance id and name tag cd /usr/bin/aws export HOME=/root +echo "setting-up aws cli env" PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) INSTANCE_NAME=$(ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') From 3e7662ae6cf1e1530cbc748eb36958b2963fc8c9 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Mon, 30 Apr 2018 18:10:05 +0530 Subject: [PATCH 37/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 4544385..cd06f5c 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -20,7 +20,6 @@ then echo "waiting for tag completion..." sleep 10 # Get instance id and name tag -cd /usr/bin/aws export HOME=/root echo "setting-up aws cli env" PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) From cb1abbedad4b23af480343438caac549412a7a3a Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Mon, 30 Apr 2018 18:34:01 +0530 Subject: [PATCH 38/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index cd06f5c..a761e8f 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -20,11 +20,10 @@ then echo "waiting for tag completion..." sleep 10 # Get instance id and name tag -export HOME=/root -echo "setting-up aws cli env" + PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) -INSTANCE_NAME=$(ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') +INSTANCE_NAME=$(/opt/aws/bin/ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') echo "${PRIVATE_IP}" | grep -q -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" From 77c7d69c5df615b3c7ce9d7bc4aed68f5439352f Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Tue, 1 May 2018 10:47:01 +0530 Subject: [PATCH 39/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index a761e8f..b950695 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -20,7 +20,7 @@ then echo "waiting for tag completion..." sleep 10 # Get instance id and name tag - +export EC2_HOME=/opt/aws/ec2-api-tools PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) INSTANCE_NAME=$(/opt/aws/bin/ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') From ef0cdbdf77646d4f004ae12c662005cf220cc210 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Tue, 1 May 2018 11:10:53 +0530 Subject: [PATCH 40/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index b950695..ffb4a58 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -20,7 +20,7 @@ then echo "waiting for tag completion..." sleep 10 # Get instance id and name tag -export EC2_HOME=/opt/aws/ec2-api-tools +export EC2_HOME=/opt/aws/apitools/ec2 PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) INSTANCE_NAME=$(/opt/aws/bin/ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') From a3037066a882d58d7537df1ffced3474babbdd57 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Tue, 1 May 2018 11:23:36 +0530 Subject: [PATCH 41/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index ffb4a58..99651c3 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -21,6 +21,7 @@ echo "waiting for tag completion..." sleep 10 # Get instance id and name tag export EC2_HOME=/opt/aws/apitools/ec2 +export JAVA_HOME=/usr/bin/java PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) INSTANCE_NAME=$(/opt/aws/bin/ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') From f804b21e4b31591d6721764b42d4db948431bf8c Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Tue, 1 May 2018 11:32:54 +0530 Subject: [PATCH 42/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 99651c3..21ed7da 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -21,7 +21,7 @@ echo "waiting for tag completion..." sleep 10 # Get instance id and name tag export EC2_HOME=/opt/aws/apitools/ec2 -export JAVA_HOME=/usr/bin/java +export JAVA_HOME=/usr/bin/ PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) INSTANCE_NAME=$(/opt/aws/bin/ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') From b488de93f992e0dd34de69664df2d06acd3a5b80 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Tue, 1 May 2018 12:26:32 +0530 Subject: [PATCH 43/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 21ed7da..d62fde0 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -21,7 +21,7 @@ echo "waiting for tag completion..." sleep 10 # Get instance id and name tag export EC2_HOME=/opt/aws/apitools/ec2 -export JAVA_HOME=/usr/bin/ +export JAVA_HOME=/usr/ PRIVATE_IP=$(curl http://169.254.169.254/latest/meta-data/local-ipv4) INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id) INSTANCE_NAME=$(/opt/aws/bin/ec2-describe-tags --region $(curl -s http://169.254.169.254/latest/meta-data/placement/availability-zone | sed -e "s/.$//") --filter resource-id=$(curl --silent http://169.254.169.254/latest/meta-data/instance-id) | head -1 | awk '{print $5}') From 9c2d2fd83868048f24460ecbbe217508a2038b85 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Tue, 1 May 2018 13:11:30 +0530 Subject: [PATCH 44/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index d62fde0..a597155 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -55,7 +55,7 @@ ARN_OF_SNS_TOPIC="arn:aws:sns:us-west-2:953030164212:SNS" CPU_USAGE=70 aws cloudwatch put-metric-alarm \ - --alarm-name "${INSTANCE_NAME}-cpu"\ + --alarm-name "{INSTANCE_ID}-${INSTANCE_NAME}-cpu"\ --alarm-description "Alarm when CPU exceeds ${CPU_USAGE}%"\ --actions-enabled\ --ok-actions "${ARN_OF_SNS_TOPIC}"\ @@ -73,7 +73,7 @@ aws cloudwatch put-metric-alarm \ # 2) Create status check metric aws cloudwatch put-metric-alarm \ - --alarm-name "${INSTANCE_NAME}-status"\ + --alarm-name "{INSTANCE_ID}-${INSTANCE_NAME}-status"\ --alarm-description "Alarm when statusCheck failed"\ --actions-enabled\ --ok-actions "${ARN_OF_SNS_TOPIC}"\ @@ -91,7 +91,7 @@ aws cloudwatch put-metric-alarm \ # 3) Create Alarm to check disk utilization aws cloudwatch put-metric-alarm \ - --alarm-name "${INSTANCE_NAME}-Disk-Utl"\ + --alarm-name "{INSTANCE_ID}-${INSTANCE_NAME}-Disk-Utl"\ --alarm-description "Alarm when Disk usage exceed 85 percent"\ --actions-enabled \ --ok-actions "${ARN_OF_SNS_TOPIC}"\ @@ -110,7 +110,7 @@ aws cloudwatch put-metric-alarm \ aws cloudwatch put-metric-alarm \ - --alarm-name "${INSTANCE_NAME}-Mem-Utl"\ + --alarm-name "{INSTANCE_ID}-${INSTANCE_NAME}-Mem-Utl"\ --alarm-description "Alarm when Memory usage exceed 80 percent"\ --actions-enabled \ --ok-actions "${ARN_OF_SNS_TOPIC}"\ From 9696c6ce1ea8c586750ab2b75af5097ffc0e23a8 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Tue, 1 May 2018 13:38:52 +0530 Subject: [PATCH 45/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index a597155..7babc86 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -126,7 +126,7 @@ aws cloudwatch put-metric-alarm \ --evaluation-periods 1\ --unit Percent -echo "CloudWatch AlarmSetup completed" +echo "CloudWatch Alarm Setup completed on $envVar" else echo "$envVar does not require CloudWatch Alarms" fi From fc7f285c083cd71a23090f754e2668aa58fd1493 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Tue, 1 May 2018 13:52:52 +0530 Subject: [PATCH 46/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 7babc86..dcd415a 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -55,7 +55,7 @@ ARN_OF_SNS_TOPIC="arn:aws:sns:us-west-2:953030164212:SNS" CPU_USAGE=70 aws cloudwatch put-metric-alarm \ - --alarm-name "{INSTANCE_ID}-${INSTANCE_NAME}-cpu"\ + --alarm-name "${INSTANCE_ID}-${INSTANCE_NAME}-cpu"\ --alarm-description "Alarm when CPU exceeds ${CPU_USAGE}%"\ --actions-enabled\ --ok-actions "${ARN_OF_SNS_TOPIC}"\ @@ -73,7 +73,7 @@ aws cloudwatch put-metric-alarm \ # 2) Create status check metric aws cloudwatch put-metric-alarm \ - --alarm-name "{INSTANCE_ID}-${INSTANCE_NAME}-status"\ + --alarm-name "${INSTANCE_ID}-${INSTANCE_NAME}-status"\ --alarm-description "Alarm when statusCheck failed"\ --actions-enabled\ --ok-actions "${ARN_OF_SNS_TOPIC}"\ @@ -91,7 +91,7 @@ aws cloudwatch put-metric-alarm \ # 3) Create Alarm to check disk utilization aws cloudwatch put-metric-alarm \ - --alarm-name "{INSTANCE_ID}-${INSTANCE_NAME}-Disk-Utl"\ + --alarm-name "${INSTANCE_ID}-${INSTANCE_NAME}-Disk-Utl"\ --alarm-description "Alarm when Disk usage exceed 85 percent"\ --actions-enabled \ --ok-actions "${ARN_OF_SNS_TOPIC}"\ @@ -110,7 +110,7 @@ aws cloudwatch put-metric-alarm \ aws cloudwatch put-metric-alarm \ - --alarm-name "{INSTANCE_ID}-${INSTANCE_NAME}-Mem-Utl"\ + --alarm-name "${INSTANCE_ID}-${INSTANCE_NAME}-Mem-Utl"\ --alarm-description "Alarm when Memory usage exceed 80 percent"\ --actions-enabled \ --ok-actions "${ARN_OF_SNS_TOPIC}"\ From 7b10c7e89c3fba0d9edd433d171512d62117b638 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Tue, 1 May 2018 15:49:28 +0530 Subject: [PATCH 47/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index dcd415a..a7cae5f 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -17,8 +17,8 @@ envVar=$1 if [ "${envVar}" = "Pre-prod" ] then -echo "waiting for tag completion..." -sleep 10 +echo "Getting Instance ID and name tag..." + # Get instance id and name tag export EC2_HOME=/opt/aws/apitools/ec2 export JAVA_HOME=/usr/ @@ -102,7 +102,7 @@ aws cloudwatch put-metric-alarm \ --statistic Maximum\ --dimensions Name=InstanceId,Value=${INSTANCE_ID}\ --period 60\ - --threshold 1\ + --threshold 85\ --comparison-operator GreaterThanOrEqualToThreshold\ --evaluation-periods 1\ --unit Percent @@ -111,7 +111,7 @@ aws cloudwatch put-metric-alarm \ aws cloudwatch put-metric-alarm \ --alarm-name "${INSTANCE_ID}-${INSTANCE_NAME}-Mem-Utl"\ - --alarm-description "Alarm when Memory usage exceed 80 percent"\ + --alarm-description "Alarm when Memory usage exceed 85 percent"\ --actions-enabled \ --ok-actions "${ARN_OF_SNS_TOPIC}"\ --alarm-actions "${ARN_OF_SNS_TOPIC}"\ @@ -121,7 +121,7 @@ aws cloudwatch put-metric-alarm \ --statistic Maximum\ --dimensions Name=InstanceId,Value=${INSTANCE_ID}\ --period 60\ - --threshold 1\ + --threshold 85\ --comparison-operator GreaterThanOrEqualToThreshold\ --evaluation-periods 1\ --unit Percent From 92fde38ebc2b7cbee12dd82bed146c61145a582a Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Tue, 1 May 2018 16:55:35 +0530 Subject: [PATCH 48/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index a7cae5f..83e6ebc 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -68,7 +68,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold ${CPU_USAGE}\ --comparison-operator GreaterThanThreshold\ - --evaluation-periods 1\ + --evaluation-periods 2\ --unit Percent # 2) Create status check metric @@ -86,7 +86,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold 1\ --comparison-operator GreaterThanOrEqualToThreshold\ - --evaluation-periods 1\ + --evaluation-periods 2\ --unit Count # 3) Create Alarm to check disk utilization @@ -104,7 +104,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold 85\ --comparison-operator GreaterThanOrEqualToThreshold\ - --evaluation-periods 1\ + --evaluation-periods 2\ --unit Percent # 4) Creat Alarm on Memory utilization @@ -123,7 +123,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold 85\ --comparison-operator GreaterThanOrEqualToThreshold\ - --evaluation-periods 1\ + --evaluation-periods 2\ --unit Percent echo "CloudWatch Alarm Setup completed on $envVar" From 2da545ba5825c707020314a73736c1352e9660bd Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Tue, 1 May 2018 18:34:55 +0530 Subject: [PATCH 49/52] Added mout path and filesystem --- scripts/cloudwatch-alarms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 83e6ebc..443d76b 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -100,7 +100,7 @@ aws cloudwatch put-metric-alarm \ --metric-name DiskSpaceUtilization\ --namespace System/Linux \ --statistic Maximum\ - --dimensions Name=InstanceId,Value=${INSTANCE_ID}\ + --dimensions Name=InstanceId,Value=${INSTANCE_ID} Name=MountPath,Value=/ Name=Filesystem,Value=/dev/xvda1\ --period 60\ --threshold 85\ --comparison-operator GreaterThanOrEqualToThreshold\ From 2e61e0e2a1ecc67b0e3bd98a5874baedf3a4966e Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Wed, 2 May 2018 10:54:59 +0530 Subject: [PATCH 50/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 443d76b..d0302f2 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -68,7 +68,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold ${CPU_USAGE}\ --comparison-operator GreaterThanThreshold\ - --evaluation-periods 2\ + --evaluation-periods 3\ --unit Percent # 2) Create status check metric @@ -86,7 +86,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold 1\ --comparison-operator GreaterThanOrEqualToThreshold\ - --evaluation-periods 2\ + --evaluation-periods 5\ --unit Count # 3) Create Alarm to check disk utilization @@ -104,7 +104,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold 85\ --comparison-operator GreaterThanOrEqualToThreshold\ - --evaluation-periods 2\ + --evaluation-periods 3\ --unit Percent # 4) Creat Alarm on Memory utilization @@ -123,7 +123,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold 85\ --comparison-operator GreaterThanOrEqualToThreshold\ - --evaluation-periods 2\ + --evaluation-periods 3\ --unit Percent echo "CloudWatch Alarm Setup completed on $envVar" From ac4590986585577c53ce23419c5ffe8103764465 Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Wed, 2 May 2018 11:32:44 +0530 Subject: [PATCH 51/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index d0302f2..0f2b7a5 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -68,7 +68,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold ${CPU_USAGE}\ --comparison-operator GreaterThanThreshold\ - --evaluation-periods 3\ + --evaluation-periods 15\ --unit Percent # 2) Create status check metric @@ -86,7 +86,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold 1\ --comparison-operator GreaterThanOrEqualToThreshold\ - --evaluation-periods 5\ + --evaluation-periods 15\ --unit Count # 3) Create Alarm to check disk utilization @@ -104,7 +104,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold 85\ --comparison-operator GreaterThanOrEqualToThreshold\ - --evaluation-periods 3\ + --evaluation-periods 15\ --unit Percent # 4) Creat Alarm on Memory utilization @@ -123,7 +123,7 @@ aws cloudwatch put-metric-alarm \ --period 60\ --threshold 85\ --comparison-operator GreaterThanOrEqualToThreshold\ - --evaluation-periods 3\ + --evaluation-periods 15\ --unit Percent echo "CloudWatch Alarm Setup completed on $envVar" From 0d4502fff540812b82b407f8aa96e7695999b7dc Mon Sep 17 00:00:00 2001 From: shamsk22 <32582082+shamsk22@users.noreply.github.com> Date: Wed, 23 May 2018 10:30:19 +0530 Subject: [PATCH 52/52] Update cloudwatch-alarms.sh --- scripts/cloudwatch-alarms.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cloudwatch-alarms.sh b/scripts/cloudwatch-alarms.sh index 0f2b7a5..3017da1 100644 --- a/scripts/cloudwatch-alarms.sh +++ b/scripts/cloudwatch-alarms.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash #=========================================================================================================================== # # FILE: create-cloudwatch-alarms.sh