From 88b5980cacd4820107829d38771401f3d47d5709 Mon Sep 17 00:00:00 2001 From: Gilles Gagniard Date: Mon, 14 Jan 2019 14:56:35 +0100 Subject: [PATCH 1/3] add ability to deploy clusters on alternative network/subnetwork --- simple/cluster.py | 1 + simple/cluster.py.schema | 4 ++++ simple/deployment.py | 2 ++ simple/deployment.py.schema | 5 +++++ simple/group.py | 4 +++- simple/group.py.schema | 9 ++++++++- 6 files changed, 23 insertions(+), 2 deletions(-) diff --git a/simple/cluster.py b/simple/cluster.py index aaa76f2..6f693d7 100644 --- a/simple/cluster.py +++ b/simple/cluster.py @@ -25,6 +25,7 @@ def GenerateConfig(context): 'license': context.properties['license'], 'cluster': context.properties['cluster'], 'region': context.properties['region'], + 'network': context.properties['network'], } for key in group: groupProperties[key] = group[key] diff --git a/simple/cluster.py.schema b/simple/cluster.py.schema index ef36dfc..6444558 100644 --- a/simple/cluster.py.schema +++ b/simple/cluster.py.schema @@ -43,3 +43,7 @@ properties: runtimeconfigName: type: string description: The runtime config resource name. Used for notifying DM about the deployment status. + + network: + type: string + description: GCP VPC network name. \ No newline at end of file diff --git a/simple/deployment.py b/simple/deployment.py index 008a4db..2e6fed9 100644 --- a/simple/deployment.py +++ b/simple/deployment.py @@ -32,6 +32,7 @@ def GenerateConfig(context): 'cluster': cluster['cluster'], 'region': cluster['region'], 'groups': cluster['groups'], + 'network': context.properties['network'], } } config['resources'].append(clusterJSON) @@ -52,6 +53,7 @@ def GenerateConfig(context): 'name': naming.FirewallName(context), 'type': 'compute.v1.firewall', 'properties': { + 'network': 'global/networks/' + context.properties['network'], 'sourceRanges': ['0.0.0.0/0'], 'allowed': [{ 'IPProtocol': 'tcp', diff --git a/simple/deployment.py.schema b/simple/deployment.py.schema index e742cf0..19abe7a 100644 --- a/simple/deployment.py.schema +++ b/simple/deployment.py.schema @@ -30,3 +30,8 @@ properties: clusters: type: array description: The list of objects, each containing configuration for a single cluster resource. + + network: + type: string + description: GCP VPC network name. + default: default \ No newline at end of file diff --git a/simple/group.py b/simple/group.py index 783eabc..70e9eea 100644 --- a/simple/group.py +++ b/simple/group.py @@ -104,7 +104,7 @@ def GenerateInstanceTemplateConfig(context, runtimeconfigName): 'properties': { 'machineType': context.properties['nodeType'], 'networkInterfaces': [{ - 'network': URL_BASE + context.env['project'] + '/global/networks/default', + 'network': 'global/networks/' + context.properties['network'], 'accessConfigs': [{ 'name': 'External NAT', 'type': 'ONE_TO_ONE_NAT' @@ -144,6 +144,8 @@ def GenerateInstanceTemplateConfig(context, runtimeconfigName): } } } + if context.properties['subnetwork']: + instanceTemplate['properties']['properties']['networkInterfaces'][0]['subnetwork'] = 'regions/' + context.properties['region'] + '/subnetworks/' + context.properties['subnetwork'] return instanceTemplate def GenerateInstanceGroupManagerConfig(context, instanceTemplateName, instanceGroupTargetSize, externalIpCreateActionName): diff --git a/simple/group.py.schema b/simple/group.py.schema index b73c1c5..5fd5225 100644 --- a/simple/group.py.schema +++ b/simple/group.py.schema @@ -28,6 +28,8 @@ properties: type: string default: 5.5.0 enum: + - 6.0.0 + - 5.5.3 - 5.5.0 - 5.1.0 - 5.0.1 @@ -80,4 +82,9 @@ properties: services: type: array description: A list of Couchbase services to be included in the instance group. - + network: + type: string + description: GCP VPC network name. + subnetwork: + type: string + description: GCP VPN subnet name (mandatory if deploying on a network in custom subnet mode). From 4cc59eb9711c3e3dbdc845b3909f00369eebf7e4 Mon Sep 17 00:00:00 2001 From: Gilles Gagniard Date: Tue, 5 Mar 2019 12:31:07 +0100 Subject: [PATCH 2/3] firewall rule: make creation optional and allow customization of source ranges --- simple/deployment.py | 31 +++++++++++++++---------------- simple/deployment.py.schema | 12 +++++++++++- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/simple/deployment.py b/simple/deployment.py index 2e6fed9..902446d 100644 --- a/simple/deployment.py +++ b/simple/deployment.py @@ -29,12 +29,11 @@ def GenerateConfig(context): 'couchbaseUsername': context.properties['couchbaseUsername'], 'couchbasePassword': context.properties['couchbasePassword'], 'license': context.properties['license'], - 'cluster': cluster['cluster'], - 'region': cluster['region'], - 'groups': cluster['groups'], 'network': context.properties['network'], } } + for key in cluster: + clusterJSON['properties'][key] = cluster[key] config['resources'].append(clusterJSON) for group in cluster['groups']: @@ -48,19 +47,19 @@ def GenerateConfig(context): 'value': '$(ref.%s.text)' % readActionName }) - - firewall = { - 'name': naming.FirewallName(context), - 'type': 'compute.v1.firewall', - 'properties': { - 'network': 'global/networks/' + context.properties['network'], - 'sourceRanges': ['0.0.0.0/0'], - 'allowed': [{ - 'IPProtocol': 'tcp', - 'ports': ['8091', '4984', '4985'] - }] + if context.properties['firewall']: + firewall = { + 'name': naming.FirewallName(context), + 'type': 'compute.v1.firewall', + 'properties': { + 'network': 'global/networks/' + context.properties['network'], + 'sourceRanges': context.properties['firewallSourceRanges'], + 'allowed': [{ + 'IPProtocol': 'tcp', + 'ports': ['8091', '4984', '4985'] + }] + } } - } - config['resources'].append(firewall) + config['resources'].append(firewall) return config diff --git a/simple/deployment.py.schema b/simple/deployment.py.schema index 19abe7a..ebf6bcf 100644 --- a/simple/deployment.py.schema +++ b/simple/deployment.py.schema @@ -34,4 +34,14 @@ properties: network: type: string description: GCP VPC network name. - default: default \ No newline at end of file + default: default + + firewall: + type: boolean + default: true + + firewallSourceRanges: + type: array + description: Allowed source address ranges for cluster + default: + - "0.0.0.0/0" From ffd819f50f3c781afc8f84dbdc9f74da3be71359 Mon Sep 17 00:00:00 2001 From: Gilles Gagniard Date: Tue, 5 Mar 2019 12:32:23 +0100 Subject: [PATCH 3/3] allow customization of memory quota for data and index services --- simple/cluster.py | 2 ++ simple/cluster.py.schema | 11 ++++++++++- simple/group.py | 2 ++ simple/group.py.schema | 10 +++++++++- simple/server.sh | 4 ++-- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/simple/cluster.py b/simple/cluster.py index 6f693d7..349c790 100644 --- a/simple/cluster.py +++ b/simple/cluster.py @@ -26,6 +26,8 @@ def GenerateConfig(context): 'cluster': context.properties['cluster'], 'region': context.properties['region'], 'network': context.properties['network'], + 'dataRAMQuotaPercent': context.properties['dataRAMQuotaPercent'], + 'indexRAMQuotaPercent': context.properties['indexRAMQuotaPercent'], } for key in group: groupProperties[key] = group[key] diff --git a/simple/cluster.py.schema b/simple/cluster.py.schema index 6444558..5b3f63c 100644 --- a/simple/cluster.py.schema +++ b/simple/cluster.py.schema @@ -46,4 +46,13 @@ properties: network: type: string - description: GCP VPC network name. \ No newline at end of file + description: GCP VPC network name. + + dataRAMQuotaPercent: + type: integer + description: Memory Quota for Data Service (percent of total memory) + default: 50 + indexRAMQuotaPercent: + type: integer + description: Memory Quota for Index Service (percent of total memory) + default: 15 \ No newline at end of file diff --git a/simple/group.py b/simple/group.py index 70e9eea..2b796f8 100644 --- a/simple/group.py +++ b/simple/group.py @@ -224,6 +224,8 @@ def GenerateStartupScript(context): script += 'couchbaseUsername="' + context.properties['couchbaseUsername'] + '"\n' script += 'couchbasePassword="' + context.properties['couchbasePassword'] + '"\n' script += 'nodeCount="' + str(context.properties['clusterNodesCount']) + '"\n' + script += 'dataRAMQuotaPercent="' + str(context.properties['dataRAMQuotaPercent']) + '"\n' + script += 'indexRAMQuotaPercent="' + str(context.properties['indexRAMQuotaPercent']) + '"\n' servicesParameter='' for service in services: diff --git a/simple/group.py.schema b/simple/group.py.schema index 5fd5225..60d8b38 100644 --- a/simple/group.py.schema +++ b/simple/group.py.schema @@ -26,8 +26,9 @@ properties: description: Type of Couchbase license to be supported by the deployment. serverVersion: type: string - default: 5.5.0 + default: 5.5.3 enum: + - 6.0.1 - 6.0.0 - 5.5.3 - 5.5.0 @@ -88,3 +89,10 @@ properties: subnetwork: type: string description: GCP VPN subnet name (mandatory if deploying on a network in custom subnet mode). + + dataRAMQuotaPercent: + type: integer + description: Memory Quota for Data Service (percent of total memory) + indexRAMQuotaPercent: + type: integer + description: Memory Quota for Index Service (percent of total memory) diff --git a/simple/server.sh b/simple/server.sh index 2732dd2..9cfa6f5 100644 --- a/simple/server.sh +++ b/simple/server.sh @@ -130,8 +130,8 @@ echo "Running couchbase-cli node-init" if [[ $rallyPrivateDNS == $NODE_PRIVATE_DNS ]] then totalRAM=$(grep MemTotal /proc/meminfo | awk '{print $2}') - dataRAM=$((50 * $totalRAM / 100000)) - indexRAM=$((15 * $totalRAM / 100000)) + dataRAM=$(($dataRAMQuotaPercent * $totalRAM / 100000)) + indexRAM=$(($indexRAMQuotaPercent * $totalRAM / 100000)) echo "Running couchbase-cli cluster-init" ./couchbase-cli cluster-init \