Creating new subnets for eks with larger pools. (#1240)

* Creating new subnets for eks with larger pools. * adding mock outputs for plamn * Converting to k8s subnet * switching from new to k8s * Renamed new to k8s * Update aws/common/vpc.tf Whoops I had the same thought and thought I did that... but I guess not! Good catch! Co-authored-by: Pat Heard <[email protected]> --------- Co-authored-by: Pat Heard <[email protected]>
cds-snc · Apr 9, 2024 · 9fddd04 · 9fddd04
1 parent f42134a
commit 9fddd04
Show file tree

Hide file tree

Showing 8 changed files with 137 additions and 1 deletion.
diff --git a/aws/common/outputs.tf b/aws/common/outputs.tf
@@ -10,6 +10,11 @@ output "vpc_private_subnets" {
   value = aws_subnet.notification-canada-ca-private.*.id
 }
 
+output "vpc_private_subnets_k8s" {
+  value = aws_subnet.notification-canada-ca-private-k8s.*.id
+}
+
+
 output "vpc_public_subnets" {
   value = aws_subnet.notification-canada-ca-public.*.id
 }

diff --git a/aws/common/s3.tf b/aws/common/s3.tf
@@ -25,6 +25,7 @@ resource "aws_s3_bucket" "csv_bucket" {
 
   #tfsec:ignore:AWS077 - Versioning is not enabled
   logging {
+    target_prefix = var.env
     target_bucket = module.csv_bucket_logs.s3_bucket_id
   }
 
@@ -155,6 +156,7 @@ resource "aws_s3_bucket" "document_bucket" {
 
   #tfsec:ignore:AWS077 - Versioning is not enabled
   logging {
+    target_prefix = var.env
     target_bucket = module.document_download_logs.s3_bucket_id
   }
 
@@ -188,6 +190,7 @@ resource "aws_s3_bucket" "scan_files_document_bucket" {
 
   #tfsec:ignore:AWS077 - Versioning is not enabled
   logging {
+    target_prefix = var.env
     target_bucket = module.document_download_logs.s3_bucket_id
   }
 
@@ -329,6 +332,7 @@ resource "aws_s3_bucket" "athena_bucket" {
 
   #tfsec:ignore:AWS077 - Versioning is not enabled
   logging {
+    target_prefix = var.env
     target_bucket = module.athena_logs_bucket.s3_bucket_id
   }
 

diff --git a/aws/common/vpc.tf b/aws/common/vpc.tf
@@ -53,6 +53,18 @@ resource "aws_eip" "notification-canada-ca-natgw" {
   }
 }
 
+resource "aws_eip" "notification-canada-ca-natgw-k8s" {
+  count      = 3
+  depends_on = [aws_internet_gateway.notification-canada-ca]
+
+  vpc = true
+
+  tags = {
+    Name       = "notification-canada-ca"
+    CostCenter = "notification-canada-ca-${var.env}"
+  }
+}
+
 resource "aws_nat_gateway" "notification-canada-ca" {
   count      = 3
   depends_on = [aws_internet_gateway.notification-canada-ca]
@@ -66,6 +78,21 @@ resource "aws_nat_gateway" "notification-canada-ca" {
   }
 }
 
+resource "aws_nat_gateway" "notification-canada-ca-k8s" {
+  count      = 3
+  depends_on = [aws_internet_gateway.notification-canada-ca]
+
+  allocation_id = aws_eip.notification-canada-ca-natgw-k8s.*.id[count.index]
+  subnet_id     = aws_subnet.notification-canada-ca-public.*.id[count.index]
+
+  tags = {
+    Name       = "notification-canada-ca"
+    CostCenter = "notification-canada-ca-${var.env}"
+  }
+}
+
+
+
 ###
 # AWS Subnets
 ###
@@ -102,6 +129,23 @@ resource "aws_subnet" "notification-canada-ca-public" {
   }
 }
 
+resource "aws_subnet" "notification-canada-ca-private-k8s" {
+  count = 3
+
+  vpc_id            = aws_vpc.notification-canada-ca.id
+  cidr_block        = cidrsubnet("10.0.0.0/16", 3, count.index + 1)
+  availability_zone = element(data.aws_availability_zones.available.names, count.index + 1)
+
+  tags = {
+    Name                                                                  = "Private Subnet 0${count.index + 4}"
+    CostCenter                                                            = "notification-canada-ca-${var.env}"
+    Access                                                                = "private"
+    "kubernetes.io/role/internal-elb"                                     = 1
+    "kubernetes.io/cluster/notification-canada-ca-${var.env}-eks-cluster" = "shared"
+    "karpenter.sh/discovery"                                              = var.eks_cluster_name
+  }
+}
+
 ###
 # AWS Routes
 ###
@@ -150,6 +194,29 @@ resource "aws_route_table_association" "notification-canada-ca-private" {
   route_table_id = aws_route_table.notification-canada-ca-private_subnet.*.id[count.index]
 }
 
+resource "aws_route_table" "notification-canada-ca-private_subnet_k8s" {
+  count = 3
+
+  vpc_id = aws_vpc.notification-canada-ca.id
+
+  route {
+    cidr_block     = "0.0.0.0/0"
+    nat_gateway_id = aws_nat_gateway.notification-canada-ca-k8s.*.id[count.index]
+  }
+
+  tags = {
+    Name       = "Private Subnet Route Table ${count.index + 3}"
+    CostCenter = "notification-canada-ca-${var.env}"
+  }
+}
+
+resource "aws_route_table_association" "notification-canada-ca-private-k8s" {
+  count = 3
+
+  subnet_id      = aws_subnet.notification-canada-ca-private-k8s.*.id[count.index]
+  route_table_id = aws_route_table.notification-canada-ca-private_subnet_k8s.*.id[count.index]
+}
+
 ###
 # AWS Network ACL
 ###

diff --git a/aws/eks/eks.tf b/aws/eks/eks.tf
@@ -93,12 +93,50 @@ resource "aws_eks_node_group" "notification-canada-ca-eks-node-group" {
   }
 }
 
+resource "aws_eks_node_group" "notification-canada-ca-eks-node-group-k8s" {
+  cluster_name         = aws_eks_cluster.notification-canada-ca-eks-cluster.name
+  node_group_name      = "notification-canada-ca-${var.env}-eks-primary-node-group-k8s"
+  node_role_arn        = aws_iam_role.eks-worker-role.arn
+  subnet_ids           = var.vpc_private_subnets_k8s
+  force_update_version = var.force_upgrade
+
+  disk_size = 80
+
+  release_version = var.eks_node_ami_version
+  instance_types  = var.primary_worker_instance_types
+
+  scaling_config {
+    desired_size = var.primary_worker_desired_size
+    max_size     = var.primary_worker_max_size
+    min_size     = var.primary_worker_min_size
+  }
+
+  update_config {
+    max_unavailable = 1
+  }
+
+  # Ensure that IAM Role permissions are created before and deleted after EKS Node Group handling.
+  # Otherwise, EKS will not be able to properly delete EC2 Instances and Elastic Network Interfaces.
+  depends_on = [
+    aws_iam_role_policy_attachment.eks-worker-AWSLoadBalancerControllerIAMPolicy,
+    aws_iam_role_policy_attachment.eks-worker-AmazonEC2ContainerRegistryReadOnly,
+    aws_iam_role_policy_attachment.eks-worker-AmazonEKSWorkerNodePolicy,
+    aws_iam_role_policy_attachment.eks-worker-AmazonEKS_CNI_Policy
+  ]
+
+  tags = {
+    Name                     = "notification-canada-ca"
+    CostCenter               = "notification-canada-ca-${var.env}"
+    "karpenter.sh/discovery" = aws_eks_cluster.notification-canada-ca-eks-cluster.name
+  }
+}
+
 resource "aws_eks_node_group" "notification-canada-ca-eks-secondary-node-group" {
   count                = var.node_upgrade ? 1 : 0
   cluster_name         = aws_eks_cluster.notification-canada-ca-eks-cluster.name
   node_group_name      = "notification-canada-ca-${var.env}-eks-secondary-node-group"
   node_role_arn        = aws_iam_role.eks-worker-role.arn
-  subnet_ids           = var.vpc_private_subnets
+  subnet_ids           = var.vpc_private_subnets_k8s
   force_update_version = var.force_upgrade
 
   disk_size = 80

diff --git a/aws/eks/variables.tf b/aws/eks/variables.tf
@@ -32,6 +32,10 @@ variable "vpc_private_subnets" {
   type = list(any)
 }
 
+variable "vpc_private_subnets_k8s" {
+  type = list(any)
+}
+
 variable "vpc_public_subnets" {
   type = list(any)
 }

diff --git a/env/dev/eks/terragrunt.hcl b/env/dev/eks/terragrunt.hcl
@@ -15,6 +15,11 @@ dependency "common" {
       "subnet-08de34a9e1a7458dc",
       "subnet-0af8b8402f1d605ff",
     ]
+    vpc_private_subnets_k8s = [
+      "subnet-001e585d12cce4d1e",
+      "subnet-08de34a9e1a7458dc",
+      "subnet-0af8b8402f1d605ff",
+    ]        
     vpc_public_subnets = [
       "subnet-0cecd9e634daf82d3",
       "subnet-0c7d18c0c51b28b61",
@@ -76,6 +81,7 @@ inputs = {
   primary_worker_min_size                   = 1
   vpc_id                                    = dependency.common.outputs.vpc_id
   vpc_private_subnets                       = dependency.common.outputs.vpc_private_subnets
+  vpc_private_subnets_k8s                   = dependency.common.outputs.vpc_private_subnets_k8s
   vpc_public_subnets                        = dependency.common.outputs.vpc_public_subnets
   sns_alert_warning_arn                     = dependency.common.outputs.sns_alert_warning_arn
   sns_alert_critical_arn                    = dependency.common.outputs.sns_alert_critical_arn

diff --git a/env/production/eks/terragrunt.hcl b/env/production/eks/terragrunt.hcl
@@ -14,6 +14,11 @@ dependency "common" {
   mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"]
   mock_outputs_merge_with_state           = true
   mock_outputs = {
+    vpc_private_subnets_k8s = [
+      "subnet-001e585d12cce4d1e",
+      "subnet-08de34a9e1a7458dc",
+      "subnet-0af8b8402f1d605ff",
+    ]        
     ip_blocklist_arn                          = ""
     re_admin_arn                              = ""
     re_api_arn                                = ""
@@ -67,6 +72,7 @@ inputs = {
   vpc_id                                    = dependency.common.outputs.vpc_id
   vpc_private_subnets                       = dependency.common.outputs.vpc_private_subnets
   vpc_public_subnets                        = dependency.common.outputs.vpc_public_subnets
+  vpc_private_subnets_k8s                   = dependency.common.outputs.vpc_private_subnets_k8s
   sns_alert_warning_arn                     = dependency.common.outputs.sns_alert_warning_arn
   sns_alert_critical_arn                    = dependency.common.outputs.sns_alert_critical_arn
   sns_alert_general_arn                     = dependency.common.outputs.sns_alert_general_arn

diff --git a/env/staging/eks/terragrunt.hcl b/env/staging/eks/terragrunt.hcl
@@ -15,6 +15,11 @@ dependency "common" {
       "subnet-08de34a9e1a7458dc",
       "subnet-0af8b8402f1d605ff",
     ]
+    vpc_private_subnets_k8s = [
+      "subnet-001e585d12cce4d1e",
+      "subnet-08de34a9e1a7458dc",
+      "subnet-0af8b8402f1d605ff",
+    ]    
     vpc_public_subnets = [
       "subnet-0cecd9e634daf82d3",
       "subnet-0c7d18c0c51b28b61",
@@ -84,6 +89,7 @@ inputs = {
   vpc_id                                    = dependency.common.outputs.vpc_id
   vpc_private_subnets                       = dependency.common.outputs.vpc_private_subnets
   vpc_public_subnets                        = dependency.common.outputs.vpc_public_subnets
+  vpc_private_subnets_k8s                   = dependency.common.outputs.vpc_private_subnets_k8s
   sns_alert_warning_arn                     = dependency.common.outputs.sns_alert_warning_arn
   sns_alert_critical_arn                    = dependency.common.outputs.sns_alert_critical_arn
   sns_alert_general_arn                     = dependency.common.outputs.sns_alert_general_arn