Update Datadog to install with flux (#972)

* Setup datadog with flux * use empty string in list * remove default value since value cant be empty * fix validation, docs and changelog * Switch to use 2 kustomizations and dependsOn for CRD installs * Add CSI and config * More config for CSI to work * Add datadog UAI config * adjust tenant id * Remove vars and use new CSI secret * Add dummypod for secret mounting, fix kv variable, and some general improvements * fix closing bracket * remove unused variable * fix vars name * Fix vars * Fix namespace name for datadogagent * Add namespace to dummy secret-mount deployment * add label for aadpodidentity * Add label to pod template * Fix so agent work * Switch name of secret * Fix agent config * Try to make it work in AWS * Add more config * Revert "Add more config" This reverts commit 8a27f73. * Revert "Try to make it work in AWS" This reverts commit b6f5770. * Revert "Revert "Try to make it work in AWS"" This reverts commit ecd65fb. * Revert "Revert "Revert "Try to make it work in AWS""" This reverts commit 8a71212. * Fix for EKS * refactor to use if cloud_provider * Update IAM for datadog * Update IAM for datadog * fix output * Try again without IAM/IRSA config * Revert "Try again without IAM/IRSA config" This reverts commit e8fa1c4. * more fixes * Fix SA name * Fix SA role_arn * Remove array from secretprovider in AWS * make docs and remove unused vars * Bump dd-operator to 0.9.2 * Bump git-provider to 0.0.2 * docs * Update flux crd * Rename datadog-agent file
XenitAB · Jun 1, 2023 · 0383e27 · 0383e27
1 parent ff85781
commit 0383e27
Show file tree

Hide file tree

Showing 29 changed files with 484 additions and 233 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## Unreleased
 
+## Changed
+
+- [#972](https://github.com/XenitAB/terraform-modules/pull/968) Update Datadog to install with flux.
+
 ## 2023.04.3
 
 ### Added

diff --git a/modules/aws/eks/README.md b/modules/aws/eks/README.md
@@ -22,6 +22,7 @@
 |------|--------|---------|
 | <a name="module_cert_manager"></a> [cert\_manager](#module\_cert\_manager) | ../irsa | n/a |
 | <a name="module_cluster_autoscaler"></a> [cluster\_autoscaler](#module\_cluster\_autoscaler) | ../irsa | n/a |
+| <a name="module_datadog"></a> [datadog](#module\_datadog) | ../irsa | n/a |
 | <a name="module_eks_ebs_csi_driver"></a> [eks\_ebs\_csi\_driver](#module\_eks\_ebs\_csi\_driver) | ../irsa | n/a |
 | <a name="module_external_dns"></a> [external\_dns](#module\_external\_dns) | ../irsa | n/a |
 | <a name="module_prometheus"></a> [prometheus](#module\_prometheus) | ../irsa | n/a |
@@ -51,6 +52,7 @@
 | [aws_eks_cluster_auth.this](https://registry.terraform.io/providers/hashicorp/aws/4.31.0/docs/data-sources/eks_cluster_auth) | data source |
 | [aws_iam_policy_document.cert_manager](https://registry.terraform.io/providers/hashicorp/aws/4.31.0/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/aws/4.31.0/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.datadog_secrets](https://registry.terraform.io/providers/hashicorp/aws/4.31.0/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.external_dns](https://registry.terraform.io/providers/hashicorp/aws/4.31.0/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.trivy_ecr_read_only](https://registry.terraform.io/providers/hashicorp/aws/4.31.0/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.velero](https://registry.terraform.io/providers/hashicorp/aws/4.31.0/docs/data-sources/iam_policy_document) | data source |
@@ -84,6 +86,7 @@
 |------|-------------|
 | <a name="output_cert_manager_config"></a> [cert\_manager\_config](#output\_cert\_manager\_config) | Configuration for Cert Manager |
 | <a name="output_cluster_autoscaler_config"></a> [cluster\_autoscaler\_config](#output\_cluster\_autoscaler\_config) | Configuration for Cluster Autoscaler |
+| <a name="output_datadog_config"></a> [datadog\_config](#output\_datadog\_config) | Configuration for Datadog |
 | <a name="output_external_dns_config"></a> [external\_dns\_config](#output\_external\_dns\_config) | Configuration for External DNS |
 | <a name="output_kube_config"></a> [kube\_config](#output\_kube\_config) | Kube config for the created EKS cluster |
 | <a name="output_prometheus_config"></a> [prometheus\_config](#output\_prometheus\_config) | Configuration for Prometheus |

diff --git a/modules/aws/eks/iam.tf b/modules/aws/eks/iam.tf
@@ -282,3 +282,37 @@ module "eks_ebs_csi_driver" {
   policy_permissions_arn     = ["arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"]
   policy_json_create         = false
 }
+
+data "aws_iam_policy_document" "datadog_secrets" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "ssm:DescribeParameters",
+    ]
+    resources = ["*"]
+  }
+  statement {
+    effect = "Allow"
+    actions = [
+      "ssm:GetParameter",
+      "ssm:GetParameters",
+    ]
+    resources = ["arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/datadog-*"]
+  }
+}
+
+module "datadog" {
+  source = "../irsa"
+
+  name = "${var.name_prefix}-${data.aws_region.current.name}-${var.environment}-${var.name}${var.eks_name_suffix}-datadog"
+  oidc_providers = [
+    {
+      url = aws_iam_openid_connect_provider.this.url
+      arn = aws_iam_openid_connect_provider.this.arn
+    }
+  ]
+  kubernetes_namespace       = "datadog"
+  kubernetes_service_account = "datadog"
+  policy_json                = data.aws_iam_policy_document.datadog_secrets.json
+  policy_json_create         = true
+}
diff --git a/modules/aws/eks/outputs.tf b/modules/aws/eks/outputs.tf
@@ -64,3 +64,9 @@ output "trivy_config" {
     trivy_role_arn          = module.trivy_ecr["trivy"].role_arn
   }
 }
+output "datadog_config" {
+  description = "Configuration for Datadog"
+  value = {
+    role_arn = module.datadog.role_arn
+  }
+}
diff --git a/modules/azure/aks-regional/README.md b/modules/azure/aks-regional/README.md
@@ -35,11 +35,13 @@ This module is used to create resources that are used by AKS clusters.
 | [azurerm_eventhub_namespace.this](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/eventhub_namespace) | resource |
 | [azurerm_eventhub_namespace_authorization_rule.aks](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/eventhub_namespace_authorization_rule) | resource |
 | [azurerm_eventhub_namespace_authorization_rule.listen](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/eventhub_namespace_authorization_rule) | resource |
+| [azurerm_key_vault_access_policy.datadog](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/key_vault_access_policy) | resource |
 | [azurerm_key_vault_access_policy.xenit](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/key_vault_access_policy) | resource |
 | [azurerm_key_vault_secret.eventhub_connection_string](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/key_vault_secret) | resource |
 | [azurerm_key_vault_secret.ssh_key](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/key_vault_secret) | resource |
 | [azurerm_public_ip_prefix.aks](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/public_ip_prefix) | resource |
 | [azurerm_role_assignment.aad_pod_identity](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/role_assignment) | resource |
+| [azurerm_role_assignment.datadog](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/role_assignment) | resource |
 | [azurerm_role_assignment.external_dns_contributor](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/role_assignment) | resource |
 | [azurerm_role_assignment.external_dns_msi](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/role_assignment) | resource |
 | [azurerm_role_assignment.external_dns_rg_read](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/role_assignment) | resource |
@@ -53,6 +55,7 @@ This module is used to create resources that are used by AKS clusters.
 | [azurerm_storage_container.velero](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/storage_container) | resource |
 | [azurerm_user_assigned_identity.aad_pod_identity](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/user_assigned_identity) | resource |
 | [azurerm_user_assigned_identity.azure_metrics](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/user_assigned_identity) | resource |
+| [azurerm_user_assigned_identity.datadog](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/user_assigned_identity) | resource |
 | [azurerm_user_assigned_identity.external_dns](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/user_assigned_identity) | resource |
 | [azurerm_user_assigned_identity.trivy](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/user_assigned_identity) | resource |
 | [azurerm_user_assigned_identity.velero](https://registry.terraform.io/providers/hashicorp/azurerm/3.51.0/docs/resources/user_assigned_identity) | resource |
@@ -99,6 +102,7 @@ This module is used to create resources that are used by AKS clusters.
 | <a name="output_aks_public_ip_prefix_ids"></a> [aks\_public\_ip\_prefix\_ids](#output\_aks\_public\_ip\_prefix\_ids) | Azure Kubernetes Service IP Prefixes |
 | <a name="output_azad_kube_proxy"></a> [azad\_kube\_proxy](#output\_azad\_kube\_proxy) | The Azure AD Application config for azad-kube-proxy |
 | <a name="output_azure_metrics_identity"></a> [azure\_metrics\_identity](#output\_azure\_metrics\_identity) | MSI authentication identity for Azure Metrics |
+| <a name="output_datadog"></a> [datadog](#output\_datadog) | MSI authentication identity for Datadog |
 | <a name="output_dns_zone"></a> [dns\_zone](#output\_dns\_zone) | DNS Zone to be used with external-dns |
 | <a name="output_external_dns_identity"></a> [external\_dns\_identity](#output\_external\_dns\_identity) | MSI authentication identity for External DNS |
 | <a name="output_log_eventhub_authorization_rule_id"></a> [log\_eventhub\_authorization\_rule\_id](#output\_log\_eventhub\_authorization\_rule\_id) | The authoritzation rule id for event hub |

diff --git a/modules/azure/aks-regional/datadog.tf b/modules/azure/aks-regional/datadog.tf
@@ -0,0 +1,19 @@
+resource "azurerm_user_assigned_identity" "datadog" {
+  resource_group_name = data.azurerm_resource_group.this.name
+  location            = data.azurerm_resource_group.this.location
+  name                = "uai-${var.environment}-${var.location_short}-${var.name}-datadog"
+}
+
+resource "azurerm_role_assignment" "datadog" {
+  scope                = azurerm_user_assigned_identity.datadog.id
+  role_definition_name = "Managed Identity Operator"
+  principal_id         = var.aks_managed_identity
+}
+
+resource "azurerm_key_vault_access_policy" "datadog" {
+  key_vault_id       = data.azurerm_key_vault.core.id
+  tenant_id          = data.azurerm_client_config.current.tenant_id
+  object_id          = azurerm_user_assigned_identity.datadog.principal_id
+  key_permissions    = local.key_vault_default_permissions.key_permissions
+  secret_permissions = local.key_vault_default_permissions.secret_permissions
+}
diff --git a/modules/azure/aks-regional/outputs.tf b/modules/azure/aks-regional/outputs.tf
@@ -108,3 +108,15 @@ output "log_eventhub_authorization_rule_id" {
   description = "The authoritzation rule id for event hub"
   value       = azurerm_eventhub_namespace_authorization_rule.aks.id
 }
+
+output "datadog" {
+  description = "MSI authentication identity for Datadog"
+  value = {
+    azure_key_vault_name = data.azurerm_key_vault.core.name
+    identity = {
+      client_id   = azurerm_user_assigned_identity.datadog.client_id
+      resource_id = azurerm_user_assigned_identity.datadog.id
+      tenant_id   = data.azurerm_client_config.current.tenant_id
+    }
+  }
+}
diff --git a/modules/kubernetes/aks-core/README.md b/modules/kubernetes/aks-core/README.md
@@ -39,7 +39,6 @@ This module is used to create AKS clusters.
 | <a name="module_csi_secrets_store_provider_azure"></a> [csi\_secrets\_store\_provider\_azure](#module\_csi\_secrets\_store\_provider\_azure) | ../../kubernetes/csi-secrets-store-provider-azure | n/a |
 | <a name="module_csi_secrets_store_provider_azure_crd"></a> [csi\_secrets\_store\_provider\_azure\_crd](#module\_csi\_secrets\_store\_provider\_azure\_crd) | ../../kubernetes/helm-crd | n/a |
 | <a name="module_datadog"></a> [datadog](#module\_datadog) | ../../kubernetes/datadog | n/a |
-| <a name="module_datadog_crd"></a> [datadog\_crd](#module\_datadog\_crd) | ../../kubernetes/helm-crd | n/a |
 | <a name="module_external_dns"></a> [external\_dns](#module\_external\_dns) | ../../kubernetes/external-dns | n/a |
 | <a name="module_falco"></a> [falco](#module\_falco) | ../../kubernetes/falco | n/a |
 | <a name="module_fluxcd_v2_azure_devops"></a> [fluxcd\_v2\_azure\_devops](#module\_fluxcd\_v2\_azure\_devops) | ../../kubernetes/fluxcd-v2-azdo | n/a |
@@ -118,7 +117,7 @@ This module is used to create AKS clusters.
 | <a name="input_control_plane_logs_config"></a> [control\_plane\_logs\_config](#input\_control\_plane\_logs\_config) | Configuration for control plane log | <pre>object({<br>    azure_key_vault_name = string<br>    identity = object({<br>      client_id   = string<br>      resource_id = string<br>      tenant_id   = string<br>    })<br>    eventhub_hostname = string<br>    eventhub_name     = string<br>  })</pre> | <pre>{<br>  "azure_key_vault_name": "",<br>  "eventhub_hostname": "",<br>  "eventhub_name": "",<br>  "identity": {<br>    "client_id": "",<br>    "resource_id": "",<br>    "tenant_id": ""<br>  }<br>}</pre> | no |
 | <a name="input_control_plane_logs_enabled"></a> [control\_plane\_logs\_enabled](#input\_control\_plane\_logs\_enabled) | Should Control plan be enabled | `bool` | `false` | no |
 | <a name="input_csi_secrets_store_provider_azure_enabled"></a> [csi\_secrets\_store\_provider\_azure\_enabled](#input\_csi\_secrets\_store\_provider\_azure\_enabled) | Should csi-secrets-store-provider-azure be enabled | `bool` | `true` | no |
-| <a name="input_datadog_config"></a> [datadog\_config](#input\_datadog\_config) | Datadog configuration | <pre>object({<br>    datadog_site         = string<br>    api_key              = string<br>    app_key              = string<br>    namespaces           = list(string)<br>    apm_ignore_resources = list(string)<br>  })</pre> | <pre>{<br>  "api_key": "",<br>  "apm_ignore_resources": [],<br>  "app_key": "",<br>  "datadog_site": "",<br>  "namespaces": [<br>    ""<br>  ]<br>}</pre> | no |
+| <a name="input_datadog_config"></a> [datadog\_config](#input\_datadog\_config) | Datadog configuration | <pre>object({<br><br>    azure_key_vault_name = string<br>    identity = object({<br>      client_id   = string<br>      resource_id = string<br>      tenant_id   = string<br>    })<br><br>    datadog_site         = string<br>    namespaces           = list(string)<br>    apm_ignore_resources = list(string)<br>  })</pre> | <pre>{<br>  "apm_ignore_resources": [<br>    ""<br>  ],<br>  "azure_key_vault_name": "",<br>  "datadog_site": "",<br>  "identity": {<br>    "client_id": "",<br>    "resource_id": "",<br>    "tenant_id": ""<br>  },<br>  "namespaces": [<br>    ""<br>  ]<br>}</pre> | no |
 | <a name="input_datadog_enabled"></a> [datadog\_enabled](#input\_datadog\_enabled) | Should Datadog be enabled | `bool` | `false` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The environment name to use for the deploy | `string` | n/a | yes |
 | <a name="input_external_dns_config"></a> [external\_dns\_config](#input\_external\_dns\_config) | External DNS configuration | <pre>object({<br>    client_id   = string<br>    resource_id = string<br>  })</pre> | n/a | yes |

diff --git a/modules/kubernetes/aks-core/modules.tf b/modules/kubernetes/aks-core/modules.tf
@@ -23,6 +23,7 @@ locals {
     "prometheus",
     "spegel",
   ]
+  cluster_id = "${var.location_short}-${var.environment}-${var.name}${local.aks_name_suffix}"
 }
 
 # OPA Gatekeeper
@@ -90,7 +91,7 @@ module "fluxcd_v2_azure_devops" {
   source = "../../kubernetes/fluxcd-v2-azdo"
 
   environment       = var.environment
-  cluster_id        = "${var.location_short}-${var.environment}-${var.name}${local.aks_name_suffix}"
+  cluster_id        = local.cluster_id
   azure_devops_pat  = var.fluxcd_v2_config.azure_devops.pat
   azure_devops_org  = var.fluxcd_v2_config.azure_devops.org
   azure_devops_proj = var.fluxcd_v2_config.azure_devops.proj
@@ -116,7 +117,7 @@ module "fluxcd_v2_github" {
   source = "../../kubernetes/fluxcd-v2-github"
 
   environment            = var.environment
-  cluster_id             = "${var.location_short}-${var.environment}-${var.name}${local.aks_name_suffix}"
+  cluster_id             = local.cluster_id
   github_org             = var.fluxcd_v2_config.github.org
   github_app_id          = var.fluxcd_v2_config.github.app_id
   github_installation_id = var.fluxcd_v2_config.github.installation_id
@@ -347,16 +348,8 @@ module "csi_secrets_store_provider_azure" {
 }
 
 # datadog
-module "datadog_crd" {
-  source = "../../kubernetes/helm-crd"
-
-  chart_repository = "https://helm.datadoghq.com"
-  chart_name       = "datadog-operator"
-  chart_version    = "0.8.0"
-}
-
 module "datadog" {
-  depends_on = [module.opa_gatekeeper, module.datadog_crd]
+  depends_on = [module.opa_gatekeeper]
 
   for_each = {
     for s in ["datadog"] :
@@ -366,13 +359,23 @@ module "datadog" {
 
   source = "../../kubernetes/datadog"
 
+  cloud_provider = "azure"
+
   location             = var.location_short
   environment          = var.environment
+  cluster_id           = local.cluster_id
   datadog_site         = var.datadog_config.datadog_site
-  api_key              = var.datadog_config.api_key
-  app_key              = var.datadog_config.app_key
   namespace_include    = var.datadog_config.namespaces
   apm_ignore_resources = var.datadog_config.apm_ignore_resources
+
+  azure_config = {
+    azure_key_vault_name = var.datadog_config.azure_key_vault_name
+    identity = {
+      client_id   = var.datadog_config.identity.client_id
+      resource_id = var.datadog_config.identity.resource_id
+      tenant_id   = data.azurerm_client_config.current.tenant_id
+    }
+  }
 }
 
 # grafana-agent

diff --git a/modules/kubernetes/aks-core/variables.tf b/modules/kubernetes/aks-core/variables.tf
@@ -284,18 +284,28 @@ variable "datadog_enabled" {
 variable "datadog_config" {
   description = "Datadog configuration"
   type = object({
+
+    azure_key_vault_name = string
+    identity = object({
+      client_id   = string
+      resource_id = string
+      tenant_id   = string
+    })
+
     datadog_site         = string
-    api_key              = string
-    app_key              = string
     namespaces           = list(string)
     apm_ignore_resources = list(string)
   })
   default = {
+    azure_key_vault_name = ""
+    identity = {
+      client_id   = ""
+      resource_id = ""
+      tenant_id   = ""
+    }
     datadog_site         = ""
-    api_key              = ""
-    app_key              = ""
     namespaces           = [""]
-    apm_ignore_resources = []
+    apm_ignore_resources = [""]
   }
 }