Merge pull request #4 from unifio/consul

Consul Integration

.vscode/settings.json

@@ -5,5 +5,6 @@
     "*.tf": "terraform"
   },
   "files.insertFinalNewline": true,
-  "files.trimTrailingWhitespace": true
+  "files.trimTrailingWhitespace": true,
+  "terraform.formatOnSave": true
 }

CHANGELOG.md

@@ -3,6 +3,17 @@
 #### IMPROVEMENTS / NEW FEATURES:
 * Add support for application autoscaling
 
+## 0.3.0 (April 26, 2017)
+
+#### BACKWARDS INCOMPATIBILITIES / NOTES:
+* Versions of Terraform prior to v0.9.0 no longer supported.
+* The following input variables have been changed:
+  * `cluster_name (string, required)` -> `cluster_label (string, required)`
+
+#### IMPROVEMENTS / NEW FEATURES:
+* Support for dpeloyment of Consul service discovery & configuration.
+* Support for deployment of Registrator for service registration with Consul.
+
 ## 0.2.0 (April 9, 2017)
 
 #### BACKWARDS INCOMPATIBILITIES / NOTES:

README.md

@@ -17,7 +17,7 @@ The cluster module provisions an ECS cluster and auto scaling group of agent ins
 #### Resource tags
 Name | Type | Required | Description
 --- | --- | --- | ---
-`cluster_name` | string | yes | Name of the ECS based application stack.
+`cluster_label` | string | yes | Short form identifier for this cluster.
 `stack_item_fullname` | string | yes | Long form descriptive name for this stack item. This value is used to create the 'application' resource tag for resources created by this stack item.
 `stack_item_label` | string | yes | Short form identifier for this stack. This value is used to create the 'Name' resource tag for resources created by this stack item, and also serves as a unique key for re-use.
 
@@ -33,27 +33,27 @@ Name | Type | Required | Description
 `ami_override` | string | | Custom Amazon Machine Image (AMI) to associate with the launch configuration.
 `associate_public_ip_address` | string | | Flag for associating public IP addresses with instances managed by the auto scaling group.
 `ebs_optimized` | string | | Flag to enable EBS optimization.
-`ebs_vol_del_on_term` | string | Default: true | Whether the volume should be destroyed on instance termination.
+`ebs_vol_del_on_term` | string | Default: `true` | Whether the volume should be destroyed on instance termination.
 `ebs_vol_device_name` | string | | The name of the device to mount.
 `ebs_vol_encrypted` | string | | Whether the volume should be encrypted or not. Do not use this option if you are using `ebs_vol_snapshot_id` as the encrypted flag will be determined by the snapshot.
-`ebs_vol_iops` | string | Default: 2000 | The amount of provisioned IOPS. Only utilized with `ebs_vol_type` of `io1`.
+`ebs_vol_iops` | string | Default: `2000` | The amount of provisioned IOPS. Only utilized with `ebs_vol_type` of `io1`.
 `ebs_vol_size` | string | | The size of the volume in gigabytes.
 `ebs_vol_snapshot_id` | string | | The Snapshot ID to mount.
-`ebs_vol_type` | string | Default: gp2 | The type of volume. Valid values are `standard`, `gp2` and `io1`.
+`ebs_vol_type` | string | Default: `gp2` | The type of volume. Valid values are `standard`, `gp2` and `io1`.
 `enable_monitoring` | string | | Flag to enable detailed monitoring.
-`iam_path` | string | Default: / | The path to the IAM resource.
+`iam_path` | string | Default: `/` | The path to the IAM resource.
 `instance_based_naming_enabled` | string | | Flag to enable instance-id based name tagging. Requires the AWS CLI to be installed on the instance. Currently only supports Linux based systems.
 `instance_name_prefix` | string | | String to prepend instance-id based name tags with.
 `instance_tags` | map | | Map of tags to add to instances. Requires the AWS CLI to be installed on the instance. Currently only supports Linux based systems.
 `instance_type` | string | yes | The EC2 instance type to associate with the launch configuration.
 `key_name` | string | | The SSH key pair to associate with the launch configuration.
-`logs_bucket_enabled` | string | Default: false | Flag for enabling access to the logs bucket from the instances.
+`logs_bucket_enabled` | string | Default: `false` | Flag for enabling access to the logs bucket from the instances.
 `logs_bucket_name` | string | | Name of the S3 bucket for logging.
-`placement_tenancy` | string | Default: default | The tenancy of the instance. Valid values are `default` or `dedicated`.
-`root_vol_del_on_term` | string | Default: true | Whether the volume should be destroyed on instance termination.
-`root_vol_iops` | string | Default: 2000 | The amount of provisioned IOPS. Only utilized with `root_vol_type` of `io1`.
+`placement_tenancy` | string | Default: `default` | The tenancy of the instance. Valid values are `default` or `dedicated`.
+`root_vol_del_on_term` | string | Default: `true` | Whether the volume should be destroyed on instance termination.
+`root_vol_iops` | string | Default: `2000` | The amount of provisioned IOPS. Only utilized with `root_vol_type` of `io1`.
 `root_vol_size` | string | | The size of the volume in gigabytes.
-`root_vol_type` | string | Default: gp2 | The type of volume. Valid values are `standard`, `gp2` and `io1`.
+`root_vol_type` | string | Default: `gp2` | The type of volume. Valid values are `standard`, `gp2` and `io1`.
 `security_groups` | list | Default: [] | A list of security group IDs to associate with the instances.
 `spot_price` | string | | The price to use for reserving spot instances.
 `user_data_override` | string | | Custom instance initialization data to associate with the launch configuration.
@@ -64,7 +64,7 @@ Name | Type | Required | Description
 `default_cooldown` | string | | The amount of time, in seconds, after a scaling activity completes before another scaling activity can start.
 `desired_capacity` | string | | The number of Amazon EC2 instances that should be running in the group.
 `enabled_metrics` | string | Default: [] | A list of metrics to collect. The allowed values are `GroupMinSize`, `GroupMaxSize`, `GroupDesiredCapacity`, `GroupInServiceInstances`, `GroupPendingInstances`, `GroupStandbyInstances`, `GroupTerminatingInstances`, `GroupTotalInstances`.
-`force_delete` | string | Default: false | Flag to allow deletion of the auto scaling group without waiting for all instances in the pool to terminate.
+`force_delete` | string | Default: `false` | Flag to allow deletion of the auto scaling group without waiting for all instances in the pool to terminate.
 `hc_check_type` | string | | Type of health check performed by the auto scaling group. Valid values are `ELB` or `EC2`.
 `hc_grace_period` | string | | Time allowed after an instance comes into service before checking health.
 `max_size` | string | yes | The maximum number of instances allowed by the auto scaling group.
@@ -75,6 +75,23 @@ Name | Type | Required | Description
 `termination_policies` | list | Default: [] | A list of policies to decide how the instances in the auto scale group should be terminated. The allowed values are `OldestInstance`, `NewestInstance`, `OldestLaunchConfiguration`, `ClosestToNextInstanceHour`, `Default`.
 `wait_for_capacity_timeout` | string | | A maximum duration that Terraform should wait for ASG managed instances to become healthy before timing out.
 
+#### Service discovery & configuration parameters
+Name | Type | Required | Description
+--- | --- | --- | ---
+`agent_config_override` | string | | Consul agent ECS task configuration JSON.
+`agent_task_arn_override` | string | | Consul agent ECS task ARN.
+`consul_dc` | string | Default: `dc1` | Consul datacenter of the specified cluster.
+`consul_gossip_cidrs` | list | Default: `0.0.0.0/0` | CIDRs encompassing all nodes wihin the Consul datacenter.
+`lb_arn` | string | yes; if `service_discovery_enabled` is `true` | Load balancer ARN.
+`lb_sg_id` | string | yes; if `service_discovery_enabled` is `true` | Consul service endpoint security group ID.
+`registrator_config_override` | string | | Registrator ECS task configuration JSON.
+`registrator_task_arn_override` | string | | Registrator ECS task ARN.
+`server_config_override` | string | | Consul server ECS task configuration JSON.
+`server_task_arn_override` | string | | Consul server ECS task ARN.
+`server_desired_count` | string | Default: `3` | The number of Consul server containers to run.
+`service_discovery_enabled` | string | Default: `false` | Flag for the deployment of Consul service discovery and configuration.
+`service_registration_enabled` | string | Default: `false` | Flag for the deployment of Registrator service registration.
+
 ### Usage ###
 
 ```js
@@ -98,6 +115,12 @@ module "cluster" {
   # ASG parameters
   max_size = "3"
   min_size = "3"
+
+  # Service discovery parameters
+  service_discovery_enabled = true
+  service_registration_enabled = true
+  la_arn = "arn:aws:elasticloadbalancing:us-east-2:XXXXXXXXXXXX:loadbalancer/app/exmpl-cmpl/93f47d7391a68bf0"
+  lb_sg_id = "sg-xxxxxxxx"
 }
 ```
 
@@ -108,6 +131,7 @@ Name | Type | Description
 `agent_role_id` | string | ID of the ECS agent IAM role.
 `cluster_id` | string | ID of the ECS cluster.
 `cluster_name` | string | Name of the ECS cluster.
+`consul_target_group_arn` | string | ARN of the Consul server target group.
 `sg_id` | string | ID of the security group associated with the agent instances.
 
 ## Examples ##

circle.yml

@@ -12,10 +12,15 @@ test:
     # UAT
     ## Test defaults
     - bin/covalence basic:defaults:apply
-    - bin/covalence basic:destroy
     ## Test overrides
     - bin/covalence complete:overrides:apply
-    - bin/covalence complete:destroy
+    ## Test service discovery
+    - bin/covalence complete:service-discovery:apply
+    - bin/covalence complete:service-registration:apply
+    - bin/covalence complete:consul-agent:apply
+    ## Clean up
+    - bin/covalence basic:destroy
+    - bin/covalence complete:consul-agent:destroy
   post:
     # Destroy VPC
     - bin/covalence prereqs:destroy

cluster/iam.tf

@@ -1,7 +1,7 @@
 # Elastic Container Service (ECS) cluster
 
 ## Creates IAM role for agent instances
-data "aws_iam_policy_document" "role" {
+data "aws_iam_policy_document" "agent_policy" {
   statement {
     actions = ["sts:AssumeRole"]
     effect  = "Allow"
@@ -13,20 +13,20 @@ data "aws_iam_policy_document" "role" {
   }
 }
 
-resource "aws_iam_role" "role" {
-  assume_role_policy = "${data.aws_iam_policy_document.role.json}"
-  name_prefix        = "${var.cluster_name}-${var.stack_item_label}-${data.aws_region.current.name}-"
+resource "aws_iam_role" "agent_role" {
+  assume_role_policy = "${data.aws_iam_policy_document.agent_policy.json}"
+  name               = "ecs-agent-${var.cluster_label}-${var.stack_item_label}-${data.aws_region.current.name}"
   path               = "${var.iam_path}"
 }
 
-resource "aws_iam_instance_profile" "profile" {
-  name_prefix = "${var.cluster_name}-${var.stack_item_label}-${data.aws_region.current.name}-"
-  path        = "${var.iam_path}"
-  roles       = ["${aws_iam_role.role.name}"]
+resource "aws_iam_instance_profile" "agent_profile" {
+  name  = "ecs-agent-${var.cluster_label}-${var.stack_item_label}-${data.aws_region.current.name}"
+  path  = "${var.iam_path}"
+  roles = ["${aws_iam_role.agent_role.name}"]
 }
 
 ### Creates monitoring policy
-data "aws_iam_policy_document" "monitoring" {
+data "aws_iam_policy_document" "monitoring_policy" {
   statement {
     actions = [
       "cloudwatch:*",
@@ -38,29 +38,29 @@ data "aws_iam_policy_document" "monitoring" {
   }
 }
 
-resource "aws_iam_role_policy" "monitoring" {
+resource "aws_iam_role_policy" "monitoring_policy" {
   name   = "monitoring"
-  policy = "${data.aws_iam_policy_document.monitoring.json}"
-  role   = "${aws_iam_role.role.id}"
+  policy = "${data.aws_iam_policy_document.monitoring_policy.json}"
+  role   = "${aws_iam_role.agent_role.id}"
 }
 
 ### Creates resource tagging policy
-data "aws_iam_policy_document" "tagging" {
+data "aws_iam_policy_document" "tagging_policy" {
   statement {
     actions   = ["ec2:CreateTags"]
     effect    = "Allow"
     resources = ["*"]
   }
 }
 
-resource "aws_iam_role_policy" "tagging" {
+resource "aws_iam_role_policy" "tagging_policy" {
   name   = "tagging"
-  policy = "${data.aws_iam_policy_document.tagging.json}"
-  role   = "${aws_iam_role.role.id}"
+  policy = "${data.aws_iam_policy_document.tagging_policy.json}"
+  role   = "${aws_iam_role.agent_role.id}"
 }
 
 ### Creates Elastic Container Service (ECS) service policy
-data "aws_iam_policy_document" "ecs" {
+data "aws_iam_policy_document" "ecs_policy" {
   statement {
     actions = [
       "ecs:CreateCluster",
@@ -78,14 +78,14 @@ data "aws_iam_policy_document" "ecs" {
   }
 }
 
-resource "aws_iam_role_policy" "ecs" {
+resource "aws_iam_role_policy" "ecs_policy" {
   name   = "ecs"
-  policy = "${data.aws_iam_policy_document.ecs.json}"
-  role   = "${aws_iam_role.role.id}"
+  policy = "${data.aws_iam_policy_document.ecs_policy.json}"
+  role   = "${aws_iam_role.agent_role.id}"
 }
 
 ### Creates Simple Storage Service (S3) policy for logging buckets
-data "aws_iam_policy_document" "logging" {
+data "aws_iam_policy_document" "logging_policy" {
   count = "${var.logs_bucket_enabled == "true" ? "1" : "0"}"
 
   statement {
@@ -105,10 +105,10 @@ data "aws_iam_policy_document" "logging" {
   }
 }
 
-resource "aws_iam_role_policy" "logging" {
+resource "aws_iam_role_policy" "logging_policy" {
   count = "${var.logs_bucket_enabled == "true" ? "1" : "0"}"
 
   name   = "logging"
-  policy = "${data.aws_iam_policy_document.logging.json}"
-  role   = "${aws_iam_role.role.id}"
+  policy = "${data.aws_iam_policy_document.logging_policy.json}"
+  role   = "${aws_iam_role.agent_role.id}"
 }

cluster/main.tf

@@ -2,7 +2,7 @@
 
 ## Set Terraform version constraint
 terraform {
-  required_version = "> 0.8.0"
+  required_version = "> 0.9.0"
 }
 
 data "aws_region" "current" {
@@ -14,7 +14,7 @@ data "template_file" "user_data" {
   template = "${var.user_data_override != "" ? "" : file("${path.module}/templates/user_data.tpl")}"
 
   vars {
-    cluster_name     = "${var.cluster_name}"
+    cluster_label    = "${var.cluster_label}"
     stack_item_label = "${var.stack_item_label}"
   }
 }
@@ -54,7 +54,7 @@ module "cluster" {
 
   # Resource tags
   stack_item_fullname = "${var.stack_item_fullname}"
-  stack_item_label    = "${var.cluster_name}-${var.stack_item_label}"
+  stack_item_label    = "${var.cluster_label}-${var.stack_item_label}"
 
   # VPC parameters
   subnets = ["${var.subnets}"]
@@ -74,7 +74,7 @@ module "cluster" {
   enable_monitoring             = "${var.enable_monitoring}"
   instance_based_naming_enabled = "${var.instance_based_naming_enabled}"
   instance_name_prefix          = "${var.instance_name_prefix}"
-  instance_profile              = "${aws_iam_instance_profile.profile.id}"
+  instance_profile              = "${aws_iam_instance_profile.agent_profile.id}"
   instance_tags                 = "${var.instance_tags}"
   instance_type                 = "${var.instance_type}"
   key_name                      = "${var.key_name}"
@@ -104,7 +104,7 @@ module "cluster" {
 }
 
 ## Updates security groups
-resource "aws_security_group_rule" "sg_agent_egress" {
+resource "aws_security_group_rule" "agent_egress" {
   cidr_blocks       = ["0.0.0.0/0"]
   from_port         = 0
   protocol          = -1
@@ -115,5 +115,44 @@ resource "aws_security_group_rule" "sg_agent_egress" {
 
 ## Registers ECS cluster
 resource "aws_ecs_cluster" "cluster" {
-  name = "${var.cluster_name}-${var.stack_item_label}"
+  name = "${var.cluster_label}-${var.stack_item_label}"
+}
+
+## Enables Consul service discovery
+module "consul" {
+  source = "../consul"
+
+  # Resource tags
+  stack_item_fullname = "${var.stack_item_fullname}"
+  stack_item_label    = "${var.cluster_label}-${var.stack_item_label}"
+
+  # ECS parameters
+  cluster_id    = "${aws_ecs_cluster.cluster.id}"
+  cluster_name  = "${aws_ecs_cluster.cluster.name}"
+  cluster_sg_id = "${module.cluster.sg_id}"
+  iam_path      = "${var.iam_path}"
+  vpc_id        = "${var.vpc_id}"
+
+  # Service discovery parameters
+  ## TODO: Enable for auto scaling
+
+  agent_config_override         = "${var.agent_config_override}"
+  agent_desired_count           = "${((length(var.desired_capacity) > 0 ? var.desired_capacity : var.min_size) - var.server_desired_count) >= 0 ? (var.min_size - var.server_desired_count) : "0"}"
+  agent_task_arn_override       = "${var.agent_task_arn_override}"
+  consul_dc                     = "${var.consul_dc}"
+  consul_docker_image           = "${var.consul_docker_image}"
+  consul_gossip_cidrs           = ["${var.consul_gossip_cidrs}"]
+  lb_arn                        = "${var.lb_arn}"
+  lb_listener_arn               = "${var.lb_listener_arn}"
+  lb_listener_rule_priority     = "${var.lb_listener_rule_priority}"
+  lb_sg_id                      = "${var.lb_sg_id}"
+  registrator_config_override   = "${var.registrator_config_override}"
+  registrator_desired_count     = "${length(var.desired_capacity) > 0 ? var.desired_capacity : var.min_size}"
+  registrator_docker_image      = "${var.registrator_docker_image}"
+  registrator_task_arn_override = "${var.registrator_task_arn_override}"
+  server_config_override        = "${var.server_config_override}"
+  server_desired_count          = "${var.server_desired_count}"
+  server_task_arn_override      = "${var.server_task_arn_override}"
+  service_discovery_enabled     = "${(var.min_size - var.server_desired_count) < 0 ? "false" : var.service_discovery_enabled}"
+  service_registration_enabled  = "${var.service_registration_enabled}"
 }

cluster/outputs.tf

@@ -1,7 +1,7 @@
 # Outputs
 
 output "agent_role_id" {
-  value = "${aws_iam_role.role.id}"
+  value = "${aws_iam_role.agent_role.id}"
 }
 
 output "cluster_id" {
@@ -12,6 +12,10 @@ output "cluster_name" {
   value = "${aws_ecs_cluster.cluster.name}"
 }
 
+output "consul_target_group_arn" {
+  value = "${module.consul.target_group_arn}"
+}
+
 output "sg_id" {
   value = "${module.cluster.sg_id}"
 }

cluster/templates/user_data.tpl

@@ -8,7 +8,7 @@ write_files:
   - path: /etc/ecs/ecs.config
     permissions: '0644'
     content: |
-      ECS_CLUSTER=${cluster_name}-${stack_item_label}
+      ECS_CLUSTER=${cluster_label}-${stack_item_label}
 
 runcmd:
   - restart ecs