From 6ea7eb8ed222a8b62e28a5eac1b322a60a1963ac Mon Sep 17 00:00:00 2001
From: haroldsphinx <adedayoakinpelu@gmail.com>
Date: Wed, 8 Feb 2023 00:09:10 +0100
Subject: [PATCH 1/3] Phase one completed

Signed-off-by: haroldsphinx <adedayoakinpelu@gmail.com>
---
 cluster_role.tf    |  65 +++++++++++++++++
 configmap.tf       |  45 ++++++++++++
 deployment.tf      | 170 +++++++++++++++++++++++++++++++++++++++++++++
 main.tf            |  12 ++++
 providers.tf       |  17 +++++
 role-binding.tf    |  17 +++++
 role.tf            |  55 +++++++++++++++
 service-account.tf |   7 ++
 service.tf         |  23 ++++++
 variables.tf       |  83 ++++++++++++++++++++++
 10 files changed, 494 insertions(+)
 create mode 100644 cluster_role.tf
 create mode 100644 configmap.tf
 create mode 100644 deployment.tf
 create mode 100644 main.tf
 create mode 100644 providers.tf
 create mode 100644 role-binding.tf
 create mode 100644 role.tf
 create mode 100644 service-account.tf
 create mode 100644 service.tf
 create mode 100644 variables.tf

diff --git a/cluster_role.tf b/cluster_role.tf
new file mode 100644
index 0000000..302f12a
--- /dev/null
+++ b/cluster_role.tf
@@ -0,0 +1,65 @@
+resource "kubernetes_cluster_role" "nginx" {
+  metadata {
+    name   = local.app_name
+    labels = local.commonLabels
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["configmaps", "endpoints", "nodes", "pods", "secrets"]
+    verbs      = ["list", "watch"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["nodes"]
+    verbs      = ["get"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["services"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    api_groups = ["extensions", "networking.k8s.io"]
+    resources  = ["ingresses"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["events"]
+    verbs      = ["create", "patch"]
+  }
+
+  rule {
+    api_groups = ["extensions", "networking.k8s.io"]
+    resources  = ["ingresses/status"]
+    verbs      = ["update"]
+  }
+
+  rule {
+    api_groups = ["networking.k8s.io"]
+    resources  = ["ingressclasses"]
+    verbs      = ["get", "list", "watch"]
+  }
+}
+
+resource "kubernetes_cluster_role_binding" "nginx" {
+  metadata {
+    name   = local.app_name
+    labels = local.commonLabels
+  }
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = kubernetes_cluster_role.nginx.id
+  }
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.nginx.metadata.0.name
+    namespace = kubernetes_service_account.nginx.metadata.0.namespace
+  }
+}
diff --git a/configmap.tf b/configmap.tf
new file mode 100644
index 0000000..a5b1ae9
--- /dev/null
+++ b/configmap.tf
@@ -0,0 +1,45 @@
+locals {
+  default_response_headers = {
+    # set by nginx: x-request-id header or auto generated
+    "x-request-id" = "$req_id"
+  }
+}
+
+resource "kubernetes_config_map" "request_headers" {
+  metadata {
+    name      = "${local.app_name}-nginx-request-headers"
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+
+  data = merge(local.default_request_headers, try(var.nginx_extra_request_headers, {}))
+}
+
+resource "kubernetes_config_map" "response_headers" {
+  metadata {
+    name      = "${local.app_name}-nginx-response-headers"
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+
+  data = merge(local.default_response_headers, try(var.nginx_extra_response_headers, {}))
+}
+
+resource "kubernetes_config_map" "nginx_ingress" {
+  metadata {
+    name      = "${local.app_name}-nginx"
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+
+  data = merge(
+    var.nginx_configmap_entries,
+    {
+      # sent upstream
+      proxy-set-headers = "${kubernetes_config_map.request_headers.metadata.0.namespace}/${kubernetes_config_map.request_headers.metadata.0.name}"
+      # added to response
+      add-headers                 = "${kubernetes_config_map.response_headers.metadata.0.namespace}/${kubernetes_config_map.response_headers.metadata.0.name}"
+      large-client-header-buffers = "8 16k"
+    }
+  )
+}
\ No newline at end of file
diff --git a/deployment.tf b/deployment.tf
new file mode 100644
index 0000000..e36f25b
--- /dev/null
+++ b/deployment.tf
@@ -0,0 +1,170 @@
+resource "kubernetes_deployment" "nginx_ingress" {
+  wait_for_rollout = false
+
+  metadata {
+    name      = local.app_name
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+
+  spec {
+    replicas = var.replicas.min
+
+    selector {
+      match_labels = local.selectorLabels
+    }
+
+    template {
+      metadata {
+        labels = local.selectorLabels
+        annotations = {
+          "prometheus.io/port"   = var.prometheus_port
+          "prometheus.io/scrape" = var.prometheus_scrape
+        }
+      }
+
+      spec {
+        service_account_name             = kubernetes_service_account.nginx_ingress.metadata.0.name
+        termination_grace_period_seconds = 300
+        node_selector                    = var.deployment_node_selector
+        priority_class_name              = var.priority_class_name
+
+        affinity {
+          pod_anti_affinity {
+            required_during_scheduling_ignored_during_execution {
+              topology_key = "kubernetes.io/hostname"
+              label_selector {
+                match_expressions {
+                  key      = "app.kubernetes.io/app"
+                  operator = "In"
+                  values   = [local.app_name]
+                }
+              }
+            }
+          }
+          node_affinity {
+            preferred_during_scheduling_ignored_during_execution {
+              weight = 1
+              preference {
+                match_expressions {
+                  key      = "restart"
+                  operator = "In"
+                  values   = ["unlikely"]
+                }
+              }
+            }
+          }
+        }
+
+        toleration {
+          effect   = "NoSchedule"
+          key      = "onlyfor"
+          operator = "Equal"
+          value    = "highcpu"
+        }
+        toleration {
+          effect   = "NoSchedule"
+          key      = "dbonly"
+          operator = "Equal"
+          value    = "yes"
+        }
+
+        container {
+          image             = "ingress-nginx/controller:${var.nginx_ingress_controller_version}"
+          name              = "nginx-ingress-controller"
+          image_pull_policy = "IfNotPresent"
+          lifecycle {
+            pre_stop {
+              exec {
+                command = ["/wait-shutdown"]
+              }
+            }
+          }
+          args = [
+            "/nginx-ingress-controller",
+            "--configmap=$(POD_NAMESPACE)/${kubernetes_config_map.nginx_ingress.metadata.0.name}",
+            "--publish-service=$(POD_NAMESPACE)/${kubernetes_service.nginx_ingress.metadata.0.name}",
+            "--election-id=${local.app_name}-leader",
+            "--ingress-class=${var.class_name}"
+          ]
+          security_context {
+            capabilities {
+              drop = ["ALL"]
+              add  = ["NET_BIND_SERVICE"]
+            }
+            run_as_user                = 101
+            allow_privilege_escalation = true
+          }
+          env {
+            name = "POD_NAME"
+            value_from {
+              field_ref {
+                field_path = "metadata.name"
+              }
+            }
+          }
+          env {
+            name = "POD_NAMESPACE"
+            value_from {
+              field_ref {
+                field_path = "metadata.namespace"
+              }
+            }
+          }
+          env {
+            name  = "LD_PRELOAD"
+            value = "/usr/local/lib/libmimalloc.so"
+          }
+          port {
+            name           = "http"
+            container_port = 80
+            protocol       = "TCP"
+          }
+          port {
+            name           = "https"
+            container_port = 443
+            protocol       = "TCP"
+          }
+          resources {
+            limits = {
+              cpu    = var.nginx_resources.limits.cpu
+              memory = var.nginx_resources.limits.memory
+            }
+            requests = {
+              cpu    = var.nginx_resources.requests.cpu
+              memory = var.nginx_resources.requests.memory
+            }
+          }
+
+          liveness_probe {
+            http_get {
+              path = "/healthz"
+              port = 10254
+            }
+            initial_delay_seconds = 10
+            period_seconds        = 10
+            success_threshold     = 1
+            timeout_seconds       = 1
+            failure_threshold     = 5
+          }
+          readiness_probe {
+            http_get {
+              path = "/healthz"
+              port = 10254
+            }
+            initial_delay_seconds = 10
+            period_seconds        = 10
+            success_threshold     = 1
+            timeout_seconds       = 1
+            failure_threshold     = 5
+          }
+        }
+      
+      }
+    }
+  }
+
+  depends_on = [
+    kubernetes_role_binding.nginx_ingress
+  ]
+}
diff --git a/main.tf b/main.tf
new file mode 100644
index 0000000..53ca88c
--- /dev/null
+++ b/main.tf
@@ -0,0 +1,12 @@
+locals {
+  app_name = "nginx-ingress-controller"
+  labels = {
+    "app.kubernetes.io/app"        = local.app_name
+    "app.kubernetes.io/owner"      = "sre"
+    "app.kubernetes.io/managed-by" = "Terraform"
+  }
+  selectorLabels = {
+    "app.kubernetes.io/app" = local.app_name
+  }
+}
+
diff --git a/providers.tf b/providers.tf
new file mode 100644
index 0000000..39a878b
--- /dev/null
+++ b/providers.tf
@@ -0,0 +1,17 @@
+
+terraform {
+  required_version = ">= 1.0.0"
+
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.10"
+    }
+  }
+}
+
+data "kubernetes_namespace" "ns" {
+  metadata {
+    name = var.namespace
+  }
+}
\ No newline at end of file
diff --git a/role-binding.tf b/role-binding.tf
new file mode 100644
index 0000000..aeb932a
--- /dev/null
+++ b/role-binding.tf
@@ -0,0 +1,17 @@
+resource "kubernetes_role_binding" "nginx_ingress" {
+  metadata {
+    name      = local.app_name
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "Role"
+    name      = kubernetes_role.nginx_ingress.metadata.0.name
+  }
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.nginx_ingress.metadata.0.name
+    namespace = kubernetes_service_account.nginx_ingress.metadata.0.namespace
+  }
+}
\ No newline at end of file
diff --git a/role.tf b/role.tf
new file mode 100644
index 0000000..09de812
--- /dev/null
+++ b/role.tf
@@ -0,0 +1,55 @@
+resource "kubernetes_role" "nginx_ingress" {
+  metadata {
+    name      = local.app_name
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["namespaces"]
+    verbs      = ["get"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["configmaps", "pods", "secrets", "endpoints"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["services"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    api_groups = ["extensions", "networking.k8s.io"]
+    resources  = ["ingresses"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    api_groups = ["extensions", "networking.k8s.io"]
+    resources  = ["ingresses/status"]
+    verbs      = ["update"]
+  }
+
+  rule {
+    api_groups = ["networking.k8s.io"]
+    resources  = ["ingressclasses"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["configmaps"]
+    verbs      = ["create", "update"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["events"]
+    verbs      = ["create", "patch"]
+  }
+}
\ No newline at end of file
diff --git a/service-account.tf b/service-account.tf
new file mode 100644
index 0000000..0641304
--- /dev/null
+++ b/service-account.tf
@@ -0,0 +1,7 @@
+resource "kubernetes_service_account" "nginx_ingress" {
+  metadata {
+    name      = local.app_name
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+}
diff --git a/service.tf b/service.tf
new file mode 100644
index 0000000..624b74c
--- /dev/null
+++ b/service.tf
@@ -0,0 +1,23 @@
+resource "kubernetes_service" "nginx_ingress" {
+  metadata {
+    name      = local.app_name
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+
+  spec {
+    selector = local.labels
+
+    port {
+      name        = "http"
+      port        = 80
+      target_port = "http"
+    }
+
+    port {
+      name        = "https"
+      port        = 443
+      target_port = "https"
+    }
+  }
+}
diff --git a/variables.tf b/variables.tf
new file mode 100644
index 0000000..bc6bc05
--- /dev/null
+++ b/variables.tf
@@ -0,0 +1,83 @@
+variable "namespace" {
+  description = "Deployment namespace"
+  type        = string
+}
+
+variable "replicas" {
+  description = "Number of deployment replicas"
+  type = object({
+    max = number
+    min = number
+  })
+  default = {
+    max = 12
+    min = 3
+  }
+}
+
+variable "prometheus_scrape" {
+  description = "Enable prometheus scraping"
+  type        = bool
+  default     = true
+}
+
+variable "prometheus_port" {
+  description = "Set prometheus port"
+  type        = number
+  default     = 10254
+}
+
+
+variable "nginx_configmap_entries" {
+  description = "Key-value pairs to be added to nginx configmap"
+  type        = map(string)
+  default     = {}
+}
+
+variable "nginx_extra_request_headers" {
+  description = "Additional nginx request headers"
+  type        = map(string)
+  default     = {}
+}
+
+variable "nginx_extra_response_headers" {
+  description = "Additional nginx response headers"
+  type        = map(string)
+  default     = {}
+}
+
+variable "nginx_image" {
+  description = "Nginx container image"
+  type = object({
+    base    = string
+    version = string
+  })
+  default = {
+    base    = "k8s.gcr.io/ingress-nginx/controller"
+    version = "v1.0.3"
+  }
+}
+
+variable "nginx_resources" {
+  description = "Nginx container resource configuration"
+  type = object({
+    limits = object({
+      cpu    = string
+      memory = string
+    })
+    requests = object({
+      cpu    = string
+      memory = string
+    })
+  })
+  default = {
+    limits = {
+      cpu    = "2"
+      memory = "1G"
+    }
+    requests = {
+      cpu    = "1"
+      memory = "512M"
+    }
+  }
+}

From ea11dbf66202b24def0f068fd2feb71949ea0c84 Mon Sep 17 00:00:00 2001
From: haroldsphinx <adedayoakinpelu@gmail.com>
Date: Sat, 18 Feb 2023 16:31:16 +0100
Subject: [PATCH 2/3] Standardize deployment

Signed-off-by: haroldsphinx <adedayoakinpelu@gmail.com>
---
 cluster_role.tf | 14 +++++++-------
 configmap.tf    |  5 +++++
 deployment.tf   |  4 ++--
 variables.tf    | 29 +++++++++++++++++++++--------
 4 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/cluster_role.tf b/cluster_role.tf
index 302f12a..7886585 100644
--- a/cluster_role.tf
+++ b/cluster_role.tf
@@ -1,7 +1,7 @@
-resource "kubernetes_cluster_role" "nginx" {
+resource "kubernetes_cluster_role" "nginx_ingress" {
   metadata {
     name   = local.app_name
-    labels = local.commonLabels
+    labels = local.labels
   }
 
   rule {
@@ -47,19 +47,19 @@ resource "kubernetes_cluster_role" "nginx" {
   }
 }
 
-resource "kubernetes_cluster_role_binding" "nginx" {
+resource "kubernetes_cluster_role_binding" "nginx_ingress" {
   metadata {
     name   = local.app_name
-    labels = local.commonLabels
+    labels = local.labels
   }
   role_ref {
     api_group = "rbac.authorization.k8s.io"
     kind      = "ClusterRole"
-    name      = kubernetes_cluster_role.nginx.id
+    name      = kubernetes_cluster_role.nginx_ingress.id
   }
   subject {
     kind      = "ServiceAccount"
-    name      = kubernetes_service_account.nginx.metadata.0.name
-    namespace = kubernetes_service_account.nginx.metadata.0.namespace
+    name      = kubernetes_service_account.nginx_ingress.metadata.0.name
+    namespace = kubernetes_service_account.nginx_ingress.metadata.0.namespace
   }
 }
diff --git a/configmap.tf b/configmap.tf
index a5b1ae9..5d7168b 100644
--- a/configmap.tf
+++ b/configmap.tf
@@ -3,6 +3,11 @@ locals {
     # set by nginx: x-request-id header or auto generated
     "x-request-id" = "$req_id"
   }
+
+  default_request_headers = {
+    "true-client-ip" = "$http_true_client_ip"
+    "x-request-id" = "$req_id"
+  }
 }
 
 resource "kubernetes_config_map" "request_headers" {
diff --git a/deployment.tf b/deployment.tf
index e36f25b..9789b0a 100644
--- a/deployment.tf
+++ b/deployment.tf
@@ -70,7 +70,7 @@ resource "kubernetes_deployment" "nginx_ingress" {
         }
 
         container {
-          image             = "ingress-nginx/controller:${var.nginx_ingress_controller_version}"
+          image             = "${var.nginx_image}"
           name              = "nginx-ingress-controller"
           image_pull_policy = "IfNotPresent"
           lifecycle {
@@ -85,7 +85,7 @@ resource "kubernetes_deployment" "nginx_ingress" {
             "--configmap=$(POD_NAMESPACE)/${kubernetes_config_map.nginx_ingress.metadata.0.name}",
             "--publish-service=$(POD_NAMESPACE)/${kubernetes_service.nginx_ingress.metadata.0.name}",
             "--election-id=${local.app_name}-leader",
-            "--ingress-class=${var.class_name}"
+            "--ingress-class=nginx"
           ]
           security_context {
             capabilities {
diff --git a/variables.tf b/variables.tf
index bc6bc05..0a2c858 100644
--- a/variables.tf
+++ b/variables.tf
@@ -48,14 +48,8 @@ variable "nginx_extra_response_headers" {
 
 variable "nginx_image" {
   description = "Nginx container image"
-  type = object({
-    base    = string
-    version = string
-  })
-  default = {
-    base    = "k8s.gcr.io/ingress-nginx/controller"
-    version = "v1.0.3"
-  }
+  type = string
+  default = "k8s.gcr.io/ingress-nginx/controller:v1.0.3"
 }
 
 variable "nginx_resources" {
@@ -81,3 +75,22 @@ variable "nginx_resources" {
     }
   }
 }
+
+
+variable "nginx_ingress_controller_image_tag" {
+  description = "The image tag to use for the NGINX ingress controller. See https://github.com/kubernetes/ingress-nginx/releases for available versions"
+  type        = string
+  default     = "v0.44.0@sha256:3dd0fac48073beaca2d67a78c746c7593f9c575168a17139a9955a82c63c4b9a"
+}
+
+variable "deployment_node_selector" {
+  description = "Map of label names and values to assign the podspec's nodeSelector property"
+  type        = map(string)
+  default = null
+}
+
+variable "priority_class_name" {
+  description = "The priority class to attach to the deployment"
+  type        = string
+  default     = null
+}
\ No newline at end of file

From 76408c22f2dd1b672619d7e25b4d8f9f04f8f2b4 Mon Sep 17 00:00:00 2001
From: haroldsphinx <adedayoakinpelu@gmail.com>
Date: Sat, 18 Feb 2023 16:32:07 +0100
Subject: [PATCH 3/3] Standardize deployment

Signed-off-by: haroldsphinx <adedayoakinpelu@gmail.com>
---
 README.md | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d3268c4..18aeef1 100644
--- a/README.md
+++ b/README.md
@@ -1 +1,12 @@
-# terraform-ingress-controller
\ No newline at end of file
+# terraform-ingress-controller
+
+## Example Usage
+
+module "ingress_controller" { 
+  source                  = "git@github.com:blockopsnetwork/terraform-ingress-controller?ref=ingress-controller-setup"
+  namespace               = "sre"
+  replicas = {
+    min = 1
+    max = 3
+  }
+}
\ No newline at end of file