Merge pull request #1 from blockopsnetwork/ingress-controller-setup

Phase one completed

README.md

@@ -1 +1,12 @@
-# terraform-ingress-controller
+# terraform-ingress-controller
+
+## Example Usage
+
+module "ingress_controller" { 
+  source                  = "[email protected]:blockopsnetwork/terraform-ingress-controller?ref=ingress-controller-setup"
+  namespace               = "sre"
+  replicas = {
+    min = 1
+    max = 3
+  }
+}

cluster_role.tf

@@ -0,0 +1,65 @@
+resource "kubernetes_cluster_role" "nginx_ingress" {
+  metadata {
+    name   = local.app_name
+    labels = local.labels
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["configmaps", "endpoints", "nodes", "pods", "secrets"]
+    verbs      = ["list", "watch"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["nodes"]
+    verbs      = ["get"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["services"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    api_groups = ["extensions", "networking.k8s.io"]
+    resources  = ["ingresses"]
+    verbs      = ["get", "list", "watch"]
+  }
+
+  rule {
+    api_groups = [""]
+    resources  = ["events"]
+    verbs      = ["create", "patch"]
+  }
+
+  rule {
+    api_groups = ["extensions", "networking.k8s.io"]
+    resources  = ["ingresses/status"]
+    verbs      = ["update"]
+  }
+
+  rule {
+    api_groups = ["networking.k8s.io"]
+    resources  = ["ingressclasses"]
+    verbs      = ["get", "list", "watch"]
+  }
+}
+
+resource "kubernetes_cluster_role_binding" "nginx_ingress" {
+  metadata {
+    name   = local.app_name
+    labels = local.labels
+  }
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "ClusterRole"
+    name      = kubernetes_cluster_role.nginx_ingress.id
+  }
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.nginx_ingress.metadata.0.name
+    namespace = kubernetes_service_account.nginx_ingress.metadata.0.namespace
+  }
+}

configmap.tf

@@ -0,0 +1,50 @@
+locals {
+  default_response_headers = {
+    # set by nginx: x-request-id header or auto generated
+    "x-request-id" = "$req_id"
+  }
+
+  default_request_headers = {
+    "true-client-ip" = "$http_true_client_ip"
+    "x-request-id" = "$req_id"
+  }
+}
+
+resource "kubernetes_config_map" "request_headers" {
+  metadata {
+    name      = "${local.app_name}-nginx-request-headers"
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+
+  data = merge(local.default_request_headers, try(var.nginx_extra_request_headers, {}))
+}
+
+resource "kubernetes_config_map" "response_headers" {
+  metadata {
+    name      = "${local.app_name}-nginx-response-headers"
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+
+  data = merge(local.default_response_headers, try(var.nginx_extra_response_headers, {}))
+}
+
+resource "kubernetes_config_map" "nginx_ingress" {
+  metadata {
+    name      = "${local.app_name}-nginx"
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+
+  data = merge(
+    var.nginx_configmap_entries,
+    {
+      # sent upstream
+      proxy-set-headers = "${kubernetes_config_map.request_headers.metadata.0.namespace}/${kubernetes_config_map.request_headers.metadata.0.name}"
+      # added to response
+      add-headers                 = "${kubernetes_config_map.response_headers.metadata.0.namespace}/${kubernetes_config_map.response_headers.metadata.0.name}"
+      large-client-header-buffers = "8 16k"
+    }
+  )
+}

deployment.tf

@@ -0,0 +1,170 @@
+resource "kubernetes_deployment" "nginx_ingress" {
+  wait_for_rollout = false
+
+  metadata {
+    name      = local.app_name
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+
+  spec {
+    replicas = var.replicas.min
+
+    selector {
+      match_labels = local.selectorLabels
+    }
+
+    template {
+      metadata {
+        labels = local.selectorLabels
+        annotations = {
+          "prometheus.io/port"   = var.prometheus_port
+          "prometheus.io/scrape" = var.prometheus_scrape
+        }
+      }
+
+      spec {
+        service_account_name             = kubernetes_service_account.nginx_ingress.metadata.0.name
+        termination_grace_period_seconds = 300
+        node_selector                    = var.deployment_node_selector
+        priority_class_name              = var.priority_class_name
+
+        affinity {
+          pod_anti_affinity {
+            required_during_scheduling_ignored_during_execution {
+              topology_key = "kubernetes.io/hostname"
+              label_selector {
+                match_expressions {
+                  key      = "app.kubernetes.io/app"
+                  operator = "In"
+                  values   = [local.app_name]
+                }
+              }
+            }
+          }
+          node_affinity {
+            preferred_during_scheduling_ignored_during_execution {
+              weight = 1
+              preference {
+                match_expressions {
+                  key      = "restart"
+                  operator = "In"
+                  values   = ["unlikely"]
+                }
+              }
+            }
+          }
+        }
+
+        toleration {
+          effect   = "NoSchedule"
+          key      = "onlyfor"
+          operator = "Equal"
+          value    = "highcpu"
+        }
+        toleration {
+          effect   = "NoSchedule"
+          key      = "dbonly"
+          operator = "Equal"
+          value    = "yes"
+        }
+
+        container {
+          image             = "${var.nginx_image}"
+          name              = "nginx-ingress-controller"
+          image_pull_policy = "IfNotPresent"
+          lifecycle {
+            pre_stop {
+              exec {
+                command = ["/wait-shutdown"]
+              }
+            }
+          }
+          args = [
+            "/nginx-ingress-controller",
+            "--configmap=$(POD_NAMESPACE)/${kubernetes_config_map.nginx_ingress.metadata.0.name}",
+            "--publish-service=$(POD_NAMESPACE)/${kubernetes_service.nginx_ingress.metadata.0.name}",
+            "--election-id=${local.app_name}-leader",
+            "--ingress-class=nginx"
+          ]
+          security_context {
+            capabilities {
+              drop = ["ALL"]
+              add  = ["NET_BIND_SERVICE"]
+            }
+            run_as_user                = 101
+            allow_privilege_escalation = true
+          }
+          env {
+            name = "POD_NAME"
+            value_from {
+              field_ref {
+                field_path = "metadata.name"
+              }
+            }
+          }
+          env {
+            name = "POD_NAMESPACE"
+            value_from {
+              field_ref {
+                field_path = "metadata.namespace"
+              }
+            }
+          }
+          env {
+            name  = "LD_PRELOAD"
+            value = "/usr/local/lib/libmimalloc.so"
+          }
+          port {
+            name           = "http"
+            container_port = 80
+            protocol       = "TCP"
+          }
+          port {
+            name           = "https"
+            container_port = 443
+            protocol       = "TCP"
+          }
+          resources {
+            limits = {
+              cpu    = var.nginx_resources.limits.cpu
+              memory = var.nginx_resources.limits.memory
+            }
+            requests = {
+              cpu    = var.nginx_resources.requests.cpu
+              memory = var.nginx_resources.requests.memory
+            }
+          }
+
+          liveness_probe {
+            http_get {
+              path = "/healthz"
+              port = 10254
+            }
+            initial_delay_seconds = 10
+            period_seconds        = 10
+            success_threshold     = 1
+            timeout_seconds       = 1
+            failure_threshold     = 5
+          }
+          readiness_probe {
+            http_get {
+              path = "/healthz"
+              port = 10254
+            }
+            initial_delay_seconds = 10
+            period_seconds        = 10
+            success_threshold     = 1
+            timeout_seconds       = 1
+            failure_threshold     = 5
+          }
+        }
+
+      }
+    }
+  }
+
+  depends_on = [
+    kubernetes_role_binding.nginx_ingress
+  ]
+}

main.tf

@@ -0,0 +1,12 @@
+locals {
+  app_name = "nginx-ingress-controller"
+  labels = {
+    "app.kubernetes.io/app"        = local.app_name
+    "app.kubernetes.io/owner"      = "sre"
+    "app.kubernetes.io/managed-by" = "Terraform"
+  }
+  selectorLabels = {
+    "app.kubernetes.io/app" = local.app_name
+  }
+}
+

providers.tf

@@ -0,0 +1,17 @@
+
+terraform {
+  required_version = ">= 1.0.0"
+
+  required_providers {
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = ">= 2.10"
+    }
+  }
+}
+
+data "kubernetes_namespace" "ns" {
+  metadata {
+    name = var.namespace
+  }
+}

role-binding.tf

@@ -0,0 +1,17 @@
+resource "kubernetes_role_binding" "nginx_ingress" {
+  metadata {
+    name      = local.app_name
+    namespace = data.kubernetes_namespace.ns.metadata.0.name
+    labels    = local.labels
+  }
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "Role"
+    name      = kubernetes_role.nginx_ingress.metadata.0.name
+  }
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.nginx_ingress.metadata.0.name
+    namespace = kubernetes_service_account.nginx_ingress.metadata.0.namespace
+  }
+}