Merge pull request #76 from equinix/v1.8.0

Anthos 1.8.3, CCM 3.2.2, and more!
equinix · Sep 30, 2021 · 79169cd · 79169cd
2 parents e2d8891 + 7a79b76
commit 79169cd
Show file tree

Hide file tree

Showing 14 changed files with 152 additions and 186 deletions.
diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl
diff --git a/README.md b/README.md
@@ -182,10 +182,9 @@ A complete list of variables can be found at <https://registry.terraform.io/modu
 |     gcp_keys_path      | string  |             n/a             | The path to a directory with GCP service account keys   |
 |        bgp_asn         | string  |            65000            | BGP ASN to peer with Equinix Metal                      |
 |      ccm_version       | string  |           v2.0.0            | The version of the Equinix Metal CCM                    |
-|    kube_vip_version    | string  |            0.2.3            | The version of Kube-VIP to install                      |
+|    kube_vip_version    | string  |            0.3.8            | The version of Kube-VIP to install                      |
 |     anthos_version     | string  |            1.7.0            | The version of Google Anthos to install                 |
 |     ccm_deploy_url     | string  | **Too Long to put here...** | The deploy url for the Equinix Metal CCM                |
-| kube_vip_daemonset_url | string  | **Too Long to put here...** | The deploy url for the Kube-VIP Daemonset               |
 |    storage_provider    | string  |             n/a             | Enable a Storage module (examples: "portworx", "rook")  |
 |    storage_options     |   map   |             n/a             | Options specific to the storage module                  |
 

diff --git a/gcp-apis.tf b/gcp-apis.tf
@@ -1,15 +1,18 @@
 locals {
   enabled_apis = [
     "anthos.googleapis.com",
+    "anthosaudit.googleapis.com",
     "anthosgke.googleapis.com",
     "cloudresourcemanager.googleapis.com",
     "container.googleapis.com",
-    "iam.googleapis.com",
     "gkeconnect.googleapis.com",
-    "serviceusage.googleapis.com",
-    "stackdriver.googleapis.com",
+    "gkehub.googleapis.com",
+    "iam.googleapis.com",
+    "opsconfigmonitoring.googleapis.com",
+    "logging.googleapis.com",
     "monitoring.googleapis.com",
-    "logging.googleapis.com"
+    "serviceusage.googleapis.com",
+    "stackdriver.googleapis.com"
   ]
 }
 

diff --git a/gcp-service-accts.tf b/gcp-service-accts.tf
@@ -70,6 +70,13 @@ resource "google_project_iam_member" "cloud_ops_sa_role_dashboard" {
   member = format("%s:%s", local.sa_text, google_service_account.cloud_ops_sa[count.index].email)
 }
 
+resource "google_project_iam_member" "cloud_ops_sa_role_metadata_writer" {
+  count  = local.sa_count
+  role   = "roles/opsconfigmonitoring.resourceMetadata.writer"
+  member = format("%s:%s", local.sa_text, google_service_account.cloud_ops_sa[count.index].email)
+}
+
+
 resource "google_project_iam_member" "bmctl_sa_compute" {
   count  = local.sa_count
   role   = "roles/compute.viewer"

diff --git a/main.tf b/main.tf
@@ -34,6 +34,7 @@ locals {
   register_sa_key     = var.gcp_keys_path == "" ? base64decode(google_service_account_key.register_sa_key[0].private_key) : file("${var.gcp_keys_path}/register.json")
   cloud_ops_sa_key    = var.gcp_keys_path == "" ? base64decode(google_service_account_key.cloud_ops_sa_key[0].private_key) : file("${var.gcp_keys_path}/cloud-ops.json")
   bmctl_sa_key        = var.gcp_keys_path == "" ? base64decode(google_service_account_key.bmctl_sa_key[0].private_key) : file("${var.gcp_keys_path}/bmctl.json")
+  ccm_deploy_url      = format("https://github.com/equinix/cloud-provider-equinix-metal/releases/download/%s/deployment.yaml", var.ccm_version)
 }
 
 resource "tls_private_key" "ssh_key_pair" {
@@ -140,7 +141,9 @@ data "template_file" "deploy_anthos_cluster" {
     cp_vip           = cidrhost(metal_reserved_ip_block.cp_vip.cidr_notation, 0)
     ingress_vip      = cidrhost(metal_reserved_ip_block.ingress_vip.cidr_notation, 0)
     cp_ips           = join(" ", metal_device.control_plane.*.access_private_ipv4)
+    cp_ids           = join(" ", metal_device.control_plane.*.id)
     worker_ips       = join(" ", metal_device.worker_nodes.*.access_private_ipv4)
+    worker_ids       = join(" ", metal_device.worker_nodes.*.id)
     anthos_ver       = var.anthos_version
   }
 }
@@ -289,8 +292,10 @@ data "template_file" "add_remaining_cps" {
   template = file("${path.module}/templates/add_remaining_cps.sh")
   vars = {
     cluster_name = local.cluster_name
-    cp_2         = metal_device.control_plane.1.access_private_ipv4
-    cp_3         = metal_device.control_plane.2.access_private_ipv4
+    cp_ip_2      = metal_device.control_plane.1.access_private_ipv4
+    cp_id_2      = metal_device.control_plane.1.id
+    cp_ip_3      = metal_device.control_plane.2.access_private_ipv4
+    cp_id_3      = metal_device.control_plane.2.id
   }
 }
 
@@ -342,39 +347,6 @@ resource "null_resource" "kube_vip_install_remaining_cp" {
   }
 }
 
-data "template_file" "worker_kubelet_flags" {
-  template = file("${path.module}/templates/worker_kubelet_flags.sh")
-}
-
-resource "null_resource" "add_kubelet_flags_to_workers" {
-  count = var.worker_count
-  depends_on = [
-    null_resource.kube_vip_install_remaining_cp,
-    null_resource.deploy_anthos_cluster,
-    null_resource.kube_vip_install_first_cp
-  ]
-  connection {
-    type        = "ssh"
-    user        = "root"
-    private_key = chomp(tls_private_key.ssh_key_pair.private_key_pem)
-    host        = element(metal_device.worker_nodes.*.access_public_ipv4, count.index)
-  }
-  provisioner "remote-exec" {
-    inline = [
-      "mkdir -p /root/bootstrap/"
-    ]
-  }
-  provisioner "file" {
-    content     = data.template_file.worker_kubelet_flags.rendered
-    destination = "/root/bootstrap/worker_kubelet_flags.sh"
-  }
-  provisioner "remote-exec" {
-    inline = [
-      "bash /root/bootstrap/worker_kubelet_flags.sh"
-    ]
-  }
-}
-
 data "template_file" "ccm_secret" {
   template = file("${path.module}/templates/ccm_secret.yaml")
   vars = {
@@ -385,7 +357,9 @@ data "template_file" "ccm_secret" {
 
 resource "null_resource" "install_ccm" {
   depends_on = [
-    null_resource.add_kubelet_flags_to_workers
+    null_resource.kube_vip_install_remaining_cp,
+    null_resource.deploy_anthos_cluster,
+    null_resource.kube_vip_install_first_cp
   ]
   connection {
     type        = "ssh"
@@ -400,13 +374,16 @@ resource "null_resource" "install_ccm" {
   provisioner "remote-exec" {
     inline = [
       "kubectl --kubeconfig /root/baremetal/bmctl-workspace/${local.cluster_name}/${local.cluster_name}-kubeconfig apply -f /root/bootstrap/ccm_secret.yaml",
-      "kubectl --kubeconfig /root/baremetal/bmctl-workspace/${local.cluster_name}/${local.cluster_name}-kubeconfig apply -f ${var.ccm_deploy_url}"
+      "kubectl --kubeconfig /root/baremetal/bmctl-workspace/${local.cluster_name}/${local.cluster_name}-kubeconfig apply -f ${local.ccm_deploy_url}"
     ]
   }
 }
 
 data "template_file" "kube_vip_ds" {
   template = file("${path.module}/templates/kube_vip_ds.yaml")
+  vars = {
+    kube_vip_ver = var.kube_vip_version
+  }
 }
 
 resource "null_resource" "install_kube_vip_daemonset" {
@@ -447,9 +424,6 @@ resource "null_resource" "worker_pre_reqs" {
     inline = ["mkdir -p /root/bootstrap/"]
   }
 
-  # Unless /root/bootstrap/ is created in advance, this will be
-  # copied to /root/bootstrap (file)
-  # https://github.com/hashicorp/terraform/issues/16330
   provisioner "file" {
     content     = data.template_file.pre_reqs_worker.rendered
     destination = "/root/bootstrap/pre_reqs_worker.sh"
@@ -464,7 +438,7 @@ module "storage" {
   source = "./modules/storage"
 
   depends_on = [
-    null_resource.add_kubelet_flags_to_workers,
+    null_resource.install_ccm,
   ]
 
   ssh = {

diff --git a/modules/rook/variables.tf b/modules/rook/variables.tf
@@ -6,8 +6,8 @@ variable "rook_version" {
 
 variable "latest_rook_version" {
   type        = string
-  description = "The version of Rook to install"
-  default     = "v1.5.10"
+  description = "The latest version of Rook that has been tested"
+  default     = "v1.7.3"
 }
 
 variable "ssh" {

diff --git a/modules/storage/variables.tf b/modules/storage/variables.tf
@@ -1,5 +1,5 @@
 variable "storage_module" {
-  description = "The name of the Storage provider module (ex. \"portworx\")"
+  description = "The name of the Storage provider module (ex. \"rook\")"
   default     = ""
 }
 

diff --git a/templates/add_remaining_cps.sh b/templates/add_remaining_cps.sh
@@ -1,12 +1,11 @@
 #!/bin/bash
 
 CLUSTER_NAME='${cluster_name}'
-CP_2='${cp_2}'
-CP_3='${cp_3}'
+CP_IP_2='${cp_ip_2}'
+CP_ID_2='${cp_id_2}'
+CP_IP_3='${cp_ip_3}'
+CP_ID_3='${cp_id_3}'
 
-# Wait a minute for things to settle
-#echo "Waiting for 60 seconds to let the cluster settle"
-#sleep 60
 kubectl \
     --kubeconfig /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME-kubeconfig \
     -n cluster-$CLUSTER_NAME \
@@ -17,14 +16,16 @@ kubectl \
                 "op": "add",
                 "path": "/spec/controlPlane/nodePoolSpec/nodes/1",
                 "value": {
-                "address": "'$CP_2'"
+                    "address": "'$CP_IP_2'", 
+                    "providerID": "equinixmetal://'$CP_ID_2'"
                 }
             },
             {
                 "op": "add",
                 "path": "/spec/controlPlane/nodePoolSpec/nodes/2",
                 "value": {
-                "address": "'$CP_3'"
+                    "address": "'$CP_IP_3'",
+                    "providerID": "equinixmetal://'$CP_ID_3'"
                 }
             }
         ]'