From 70f4b92d19cdad9a5c9a084143113414f9ed0b95 Mon Sep 17 00:00:00 2001 From: Cody Hill Date: Thu, 16 Sep 2021 18:52:16 +0000 Subject: [PATCH 1/3] Upgraded Anthos to 1.8.3, implemented `ProviderID` and `cloud-provider=external`, upgraded to the official Equinix Metal CCM, we now need to focus on getting to the latest version of Kube-VIP --- .terraform.lock.hcl | 135 +++++++++++++++--------------- gcp-apis.tf | 11 ++- gcp-service-accts.tf | 8 ++ main.tf | 53 +++--------- modules/rook/variables.tf | 4 +- modules/storage/variables.tf | 2 +- templates/add_remaining_cps.sh | 15 ++-- templates/ccm_secret.yaml | 15 ++++ templates/kube_vip_install.sh | 3 - templates/pre_reqs.sh | 19 +++-- templates/worker_kubelet_flags.sh | 26 ------ variables.tf | 12 +-- 12 files changed, 134 insertions(+), 169 deletions(-) delete mode 100644 templates/worker_kubelet_flags.sh diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 252fe78..88d2699 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,21 +2,22 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/equinix/metal" { - version = "1.0.0" + version = "3.1.0" hashes = [ - "h1:Li/OfOSEoZpk/Dnn2vQ9T9jaWfgjAedA5kaafmi9b7E=", - "h1:nUIvAmwG1AIo5FCrqiNoB9oWHjRs5K6YTwr2T3iIEkY=", - "zh:06bf3f257e490891a9470e6b559d27b4136c2a699d05eb5690cad5fec98d7d4d", - "zh:0d1312af7461a3ee6def3f91e82ac0b06c66667423c459791cf52dda75527428", - "zh:0d866ece81ff174d5a25890ed18672e15a5ddd88834335289b62541690076ab8", - "zh:0fb605b2271883a8a76d330ca479687f8ee001f47594ff289f7aeb9df4d2265a", - "zh:2f5f6f582bd11a24cca99fbde1da31999dd8c3d1117f17ee2d5f9eb048e02100", - "zh:3d12c99b463f732eb5f4a81f38b6eeba4314892669f4ce3ced8d91cd2b9209a3", - "zh:6045a87e5b74c90fbe07352aedab1ec3175c421b36caebeaacc1f7446bede27c", - "zh:6528716f0fdd24b4eada6ca5b526f12f19bdf987e998abc2ec5942d59986be6a", - "zh:7bc75f9b60928d8b843e4342b069724e224ff051c54e606efaa1b080d25e8f6e", - "zh:b530619842716e48342fc79b04b2b261c89bc26019c9bac4993ea532b17b3b89", - "zh:c3321a23259cbdb0488a0450a85197db4a467776287e96cf67c5731d9f83c58e", + "h1:IuYQx4pH1ooN7gt689g/nIy56hEkIop/9aVa+TXulv8=", + "zh:070b77456a3de02b623be29f1752b48c17100533a98661b7c6555ad2b60aeb1b", + "zh:140b4bfb72810b2e977e7fe442a5b69096db93fe3daff6e307574a541079eb42", + "zh:1d9bf5ba1e05b55af6f5d3e37fa9c0dd4c9fed02e0d47112945c16561f10fe38", + "zh:2b380fb077bb21296e1560c37efd051a7bb31fa488fbe525f54a166ace8cccc8", + "zh:5120fd9f3164e06473d813206e4f06d3b330a351ef6f59114546e0170a49f542", + "zh:5b131619662a36af7f0691c9505d24462533965391039b5664855e50b64db7c4", + "zh:7d3b2f2e2d1a0073ae54070e5dee37713726f6401f26d046db8a5cf58c80cebc", + "zh:7d40ec64e8a3f77c272c9e0cfc8bb6fc148e11a130ef0f022291947313798ec1", + "zh:7fe35422891a14ed233080303468221d8a9f98cd74b0fe73688569cbfc19ff4c", + "zh:a9bb9bfe0928bbc15c132004b3ef573983bb14cc59af9fb492b4f59f2d159054", + "zh:bd7000a809e2e96f87ac7ef90c953a05a8813feb220f27cccc50232f1ae02a50", + "zh:c0656d79d71b893fabcc8420774b8c2d5e2c77feb2b7fecd5dfb9be24228d392", + "zh:f1c755853d4c988fc616e8b1cf8407ed425a0e9ef35a441c0e9eb37657aa9907", ] } @@ -25,7 +26,6 @@ provider "registry.terraform.io/hashicorp/google" { constraints = "~> 3.53.0" hashes = [ "h1:0MYwK1KRNCc9lfF8vV9gDEuaylwEfSPws7ZJbLwY2FE=", - "h1:AzrT8ueZHo7GrEWFiXi3eB/NOQoXcVE/w6fLcRJyc34=", "zh:1408365b5f2ae508fce9b446bb9dbaf044aec81fa4c36fff39c2511b179bcc56", "zh:1d53e978065feb6278bc8c88a70c3df7599c3b8bbcd77765bcd842a83bce6686", "zh:5173a92249c8d06d0d2beca0e328df6e956becd789ebae9a064f022151415b8f", @@ -40,63 +40,62 @@ provider "registry.terraform.io/hashicorp/google" { } provider "registry.terraform.io/hashicorp/local" { - version = "2.0.0" + version = "2.1.0" hashes = [ - "h1:EC6eh7avwx1rF56h3RZcxgEp/14ihi7Sk/4J3Hn4nIE=", - "h1:pO1ANXtOCRfecKsY9Hn4UsXoPBLv6LFiDIEiS1MZ09E=", - "zh:34ce8b79493ace8333d094752b579ccc907fa9392a2c1d6933a6c95d0786d3f1", - "zh:5c5a19c4f614a4ffb68bae0b0563f3860115cf7539b8adc21108324cfdc10092", - "zh:67ddb1ca2cd3e1a8f948302597ceb967f19d2eeb2d125303493667388fe6330e", - "zh:68e6b16f3a8e180fcba1a99754118deb2d82331b51f6cca39f04518339bfdfa6", - "zh:8393a12eb11598b2799d51c9b0a922a3d9fadda5a626b94a1b4914086d53120e", - "zh:90daea4b2010a86f2aca1e3a9590e0b3ddcab229c2bd3685fae76a832e9e836f", - "zh:99308edc734a0ac9149b44f8e316ca879b2670a1cae387a8ae754c180b57cdb4", - "zh:c76594db07a9d1a73372a073888b672df64adb455d483c2426cc220eda7e092e", - "zh:dc09c1fb36c6a706bdac96cce338952888c8423978426a09f5df93031aa88b84", - "zh:deda88134e9780319e8de91b3745520be48ead6ec38cb662694d09185c3dac70", + "h1:EYZdckuGU3n6APs97nS2LxZm3dDtGqyM4qaIvsmac8o=", + "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", + "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", + "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", + "zh:719dfd97bb9ddce99f7d741260b8ece2682b363735c764cac83303f02386075a", + "zh:7598bb86e0378fd97eaa04638c1a4c75f960f62f69d3662e6d80ffa5a89847fe", + "zh:ad0a188b52517fec9eca393f1e2c9daea362b33ae2eb38a857b6b09949a727c1", + "zh:c46846c8df66a13fee6eff7dc5d528a7f868ae0dcf92d79deaac73cc297ed20c", + "zh:dc1a20a2eec12095d04bf6da5321f535351a594a636912361db20eb2a707ccc4", + "zh:e57ab4771a9d999401f6badd8b018558357d3cbdf3d33cc0c4f83e818ca8e94b", + "zh:ebdcde208072b4b0f8d305ebf2bfdc62c926e0717599dcf8ec2fd8c5845031c3", + "zh:ef34c52b68933bedd0868a13ccfd59ff1c820f299760b3c02e008dc95e2ece91", ] } provider "registry.terraform.io/hashicorp/null" { - version = "3.0.0" + version = "3.1.0" hashes = [ - "h1:V1tzrSG6t3e7zWvUwRbGbhsWU2Jd/anrJpOl9XM+R/8=", - "h1:ysHGBhBNkIiJLEpthB/IVCLpA1Qoncp3KbCTFGFZTO0=", - "zh:05fb7eab469324c97e9b73a61d2ece6f91de4e9b493e573bfeda0f2077bc3a4c", - "zh:1688aa91885a395c4ae67636d411475d0b831e422e005dcf02eedacaafac3bb4", - "zh:24a0b1292e3a474f57c483a7a4512d797e041bc9c2fbaac42fe12e86a7fb5a3c", - "zh:2fc951bd0d1b9b23427acc93be09b6909d72871e464088171da60fbee4fdde03", - "zh:6db825759425599a326385a68acc6be2d9ba0d7d6ef587191d0cdc6daef9ac63", - "zh:85985763d02618993c32c294072cc6ec51f1692b803cb506fcfedca9d40eaec9", - "zh:a53186599c57058be1509f904da512342cfdc5d808efdaf02dec15f0f3cb039a", - "zh:c2e07b49b6efa676bdc7b00c06333ea1792a983a5720f9e2233db27323d2707c", - "zh:cdc8fe1096103cf5374751e2e8408ec4abd2eb67d5a1c5151fe2c7ecfd525bef", - "zh:dbdef21df0c012b0d08776f3d4f34eb0f2f229adfde07ff252a119e52c0f65b7", + "h1:vpC6bgUQoJ0znqIKVFevOdq+YQw42bRq0u+H3nto8nA=", + "zh:02a1675fd8de126a00460942aaae242e65ca3380b5bb192e8773ef3da9073fd2", + "zh:53e30545ff8926a8e30ad30648991ca8b93b6fa496272cd23b26763c8ee84515", + "zh:5f9200bf708913621d0f6514179d89700e9aa3097c77dac730e8ba6e5901d521", + "zh:9ebf4d9704faba06b3ec7242c773c0fbfe12d62db7d00356d4f55385fc69bfb2", + "zh:a6576c81adc70326e4e1c999c04ad9ca37113a6e925aefab4765e5a5198efa7e", + "zh:a8a42d13346347aff6c63a37cda9b2c6aa5cc384a55b2fe6d6adfa390e609c53", + "zh:c797744d08a5307d50210e0454f91ca4d1c7621c68740441cf4579390452321d", + "zh:cecb6a304046df34c11229f20a80b24b1603960b794d68361a67c5efe58e62b8", + "zh:e1371aa1e502000d9974cfaff5be4cfa02f47b17400005a16f14d2ef30dc2a70", + "zh:fc39cc1fe71234a0b0369d5c5c7f876c71b956d23d7d6f518289737a001ba69b", + "zh:fea4227271ebf7d9e2b61b89ce2328c7262acd9fd190e1fd6d15a591abfa848e", ] } provider "registry.terraform.io/hashicorp/random" { - version = "3.0.1" + version = "3.1.0" hashes = [ - "h1:0QaSbRBgBi8vI/8IRwec1INdOqBxXbgsSFElx1O4k4g=", - "h1:SzM8nt2wzLMI28A3CWAtW25g3ZCm1O4xD0h3Ps/rU1U=", - "zh:0d4f683868324af056a9eb2b06306feef7c202c88dbbe6a4ad7517146a22fb50", - "zh:4824b3c7914b77d41dfe90f6f333c7ac9860afb83e2a344d91fbe46e5dfbec26", - "zh:4b82e43712f3cf0d0cbc95b2cbcd409ba8f0dc7848fdfb7c13633c27468ed04a", - "zh:78b3a2b860c3ebc973a794000015f5946eb59b82705d701d487475406b2612f1", - "zh:88bc65197bd74ff408d147b32f0045372ae3a3f2a2fdd7f734f315d988c0e4a2", - "zh:91bd3c9f625f177f3a5d641a64e54d4b4540cb071070ecda060a8261fb6eb2ef", - "zh:a6818842b28d800f784e0c93284ff602b0c4022f407e4750da03f50b853a9a2c", - "zh:c4a1a2b52abd05687e6cfded4a789dcd7b43e7a746e4d02dd1055370cf9a994d", - "zh:cf65041bf12fc3bde709c1d267dbe94142bc05adcabc4feb17da3b12249132ac", - "zh:e385e00e7425dda9d30b74ab4ffa4636f4b8eb23918c0b763f0ffab84ece0c5c", + "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=", + "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", + "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", + "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", + "zh:7011332745ea061e517fe1319bd6c75054a314155cb2c1199a5b01fe1889a7e2", + "zh:738ed82858317ccc246691c8b85995bc125ac3b4143043219bd0437adc56c992", + "zh:7dbe52fac7bb21227acd7529b487511c91f4107db9cc4414f50d04ffc3cab427", + "zh:a3a9251fb15f93e4cfc1789800fc2d7414bbc18944ad4c5c98f466e6477c42bc", + "zh:a543ec1a3a8c20635cf374110bd2f87c07374cf2c50617eee2c669b3ceeeaa9f", + "zh:d9ab41d556a48bd7059f0810cf020500635bfc696c9fc3adab5ea8915c1d886b", + "zh:d9e13427a7d011dbd654e591b0337e6074eef8c3b9bb11b2e39eaaf257044fd7", + "zh:f7605bd1437752114baf601bdf6931debe6dc6bfe3006eb7e9bb9080931dca8a", ] } provider "registry.terraform.io/hashicorp/template" { version = "2.2.0" hashes = [ - "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", @@ -112,19 +111,19 @@ provider "registry.terraform.io/hashicorp/template" { } provider "registry.terraform.io/hashicorp/tls" { - version = "3.0.0" + version = "3.1.0" hashes = [ - "h1:AcQGOAD5xa4KE9gYw5g7R6UU8a77Yn/afPvch4N86lQ=", - "h1:LtCEW5v1E5Eo49+kQOsKHRYf9Hc8ZR0jTpK+mXszPHs=", - "zh:05eac573a1fe53227bcc6b01daf6ddf0b73456f97f56f316f1b3114a4771e175", - "zh:09390dad764c76f0fd59cae4dad296e3e39487e06de3a4bc0df73916c6bb2f17", - "zh:142d0bc4722ab088b7ca124b0eb44206b9d100f51035c162d50ef552e09813d0", - "zh:2c391743dd20f43329c0d0d49dec7827970d788115593c0e32a57050c0a85337", - "zh:525b12fc87369c0e6d347afe6c77668aebf56cfa078bb0f1f01cc2ee01ac7016", - "zh:5583d81b7a05c6d49a4c445e1ee62e82facb07bb9204998a836b7b522a51db8d", - "zh:925e3acc70e18ed1cd296d337fc3e0ca43ac6f5bf2e660f24de750c7754f91aa", - "zh:a291457d25b207fd28fb4fad9209ebb591e25cfc507ca1cb0fb8b2e255be1969", - "zh:bbf9e2718752aebfbd7c6b8e196eb2e52730b66befed2ea1954f9ff1c199295e", - "zh:f4b333c467ae02c1a238ac57465fe66405f6e2a6cfeb4eded9bc321c5652a1bf", + "h1:fUJX8Zxx38e2kBln+zWr1Tl41X+OuiE++REjrEyiOM4=", + "zh:3d46616b41fea215566f4a957b6d3a1aa43f1f75c26776d72a98bdba79439db6", + "zh:623a203817a6dafa86f1b4141b645159e07ec418c82fe40acd4d2a27543cbaa2", + "zh:668217e78b210a6572e7b0ecb4134a6781cc4d738f4f5d09eb756085b082592e", + "zh:95354df03710691773c8f50a32e31fca25f124b7f3d6078265fdf3c4e1384dca", + "zh:9f97ab190380430d57392303e3f36f4f7835c74ea83276baa98d6b9a997c3698", + "zh:a16f0bab665f8d933e95ca055b9c8d5707f1a0dd8c8ecca6c13091f40dc1e99d", + "zh:be274d5008c24dc0d6540c19e22dbb31ee6bfdd0b2cddd4d97f3cd8a8d657841", + "zh:d5faa9dce0a5fc9d26b2463cea5be35f8586ab75030e7fa4d4920cd73ee26989", + "zh:e9b672210b7fb410780e7b429975adcc76dd557738ecc7c890ea18942eb321a5", + "zh:eb1f8368573d2370605d6dbf60f9aaa5b64e55741d96b5fb026dbfe91de67c0d", + "zh:fc1e12b713837b85daf6c3bb703d7795eaf1c5177aebae1afcf811dd7009f4b0", ] } diff --git a/gcp-apis.tf b/gcp-apis.tf index 1b1d93d..df75f91 100644 --- a/gcp-apis.tf +++ b/gcp-apis.tf @@ -1,15 +1,18 @@ locals { enabled_apis = [ "anthos.googleapis.com", + "anthosaudit.googleapis.com", "anthosgke.googleapis.com", "cloudresourcemanager.googleapis.com", "container.googleapis.com", - "iam.googleapis.com", "gkeconnect.googleapis.com", - "serviceusage.googleapis.com", - "stackdriver.googleapis.com", + "gkehub.googleapis.com", + "iam.googleapis.com", + "opsconfigmonitoring.googleapis.com", + "logging.googleapis.com", "monitoring.googleapis.com", - "logging.googleapis.com" + "serviceusage.googleapis.com", + "stackdriver.googleapis.com" ] } diff --git a/gcp-service-accts.tf b/gcp-service-accts.tf index 00f791b..908fab9 100644 --- a/gcp-service-accts.tf +++ b/gcp-service-accts.tf @@ -61,6 +61,7 @@ resource "google_project_iam_member" "cloud_ops_sa_role_metricwriter" { resource "google_project_iam_member" "cloud_ops_sa_role_resourcewriter" { count = local.sa_count role = "roles/stackdriver.resourceMetadata.writer" + member = format("%s:%s", local.sa_text, google_service_account.cloud_ops_sa[count.index].email) } @@ -70,6 +71,13 @@ resource "google_project_iam_member" "cloud_ops_sa_role_dashboard" { member = format("%s:%s", local.sa_text, google_service_account.cloud_ops_sa[count.index].email) } +resource "google_project_iam_member" "cloud_ops_sa_role_metadata_writer" { + count = local.sa_count + role = "roles/opsconfigmonitoring.resourceMetadata.writer" + member = format("%s:%s", local.sa_text, google_service_account.cloud_ops_sa[count.index].email) +} + + resource "google_project_iam_member" "bmctl_sa_compute" { count = local.sa_count role = "roles/compute.viewer" diff --git a/main.tf b/main.tf index b29440e..896f22c 100644 --- a/main.tf +++ b/main.tf @@ -34,6 +34,7 @@ locals { register_sa_key = var.gcp_keys_path == "" ? base64decode(google_service_account_key.register_sa_key[0].private_key) : file("${var.gcp_keys_path}/register.json") cloud_ops_sa_key = var.gcp_keys_path == "" ? base64decode(google_service_account_key.cloud_ops_sa_key[0].private_key) : file("${var.gcp_keys_path}/cloud-ops.json") bmctl_sa_key = var.gcp_keys_path == "" ? base64decode(google_service_account_key.bmctl_sa_key[0].private_key) : file("${var.gcp_keys_path}/bmctl.json") + ccm_deploy_url = format("https://github.com/equinix/cloud-provider-equinix-metal/releases/download/%s/deployment.yaml", var.ccm_version) } resource "tls_private_key" "ssh_key_pair" { @@ -140,7 +141,9 @@ data "template_file" "deploy_anthos_cluster" { cp_vip = cidrhost(metal_reserved_ip_block.cp_vip.cidr_notation, 0) ingress_vip = cidrhost(metal_reserved_ip_block.ingress_vip.cidr_notation, 0) cp_ips = join(" ", metal_device.control_plane.*.access_private_ipv4) + cp_ids = join(" ", metal_device.control_plane.*.id) worker_ips = join(" ", metal_device.worker_nodes.*.access_private_ipv4) + worker_ids = join(" ", metal_device.worker_nodes.*.id) anthos_ver = var.anthos_version } } @@ -289,8 +292,10 @@ data "template_file" "add_remaining_cps" { template = file("${path.module}/templates/add_remaining_cps.sh") vars = { cluster_name = local.cluster_name - cp_2 = metal_device.control_plane.1.access_private_ipv4 - cp_3 = metal_device.control_plane.2.access_private_ipv4 + cp_ip_2 = metal_device.control_plane.1.access_private_ipv4 + cp_id_2 = metal_device.control_plane.1.id + cp_ip_3 = metal_device.control_plane.2.access_private_ipv4 + cp_id_3 = metal_device.control_plane.2.id } } @@ -342,39 +347,6 @@ resource "null_resource" "kube_vip_install_remaining_cp" { } } -data "template_file" "worker_kubelet_flags" { - template = file("${path.module}/templates/worker_kubelet_flags.sh") -} - -resource "null_resource" "add_kubelet_flags_to_workers" { - count = var.worker_count - depends_on = [ - null_resource.kube_vip_install_remaining_cp, - null_resource.deploy_anthos_cluster, - null_resource.kube_vip_install_first_cp - ] - connection { - type = "ssh" - user = "root" - private_key = chomp(tls_private_key.ssh_key_pair.private_key_pem) - host = element(metal_device.worker_nodes.*.access_public_ipv4, count.index) - } - provisioner "remote-exec" { - inline = [ - "mkdir -p /root/bootstrap/" - ] - } - provisioner "file" { - content = data.template_file.worker_kubelet_flags.rendered - destination = "/root/bootstrap/worker_kubelet_flags.sh" - } - provisioner "remote-exec" { - inline = [ - "bash /root/bootstrap/worker_kubelet_flags.sh" - ] - } -} - data "template_file" "ccm_secret" { template = file("${path.module}/templates/ccm_secret.yaml") vars = { @@ -385,7 +357,9 @@ data "template_file" "ccm_secret" { resource "null_resource" "install_ccm" { depends_on = [ - null_resource.add_kubelet_flags_to_workers + null_resource.kube_vip_install_remaining_cp, + null_resource.deploy_anthos_cluster, + null_resource.kube_vip_install_first_cp ] connection { type = "ssh" @@ -400,7 +374,7 @@ resource "null_resource" "install_ccm" { provisioner "remote-exec" { inline = [ "kubectl --kubeconfig /root/baremetal/bmctl-workspace/${local.cluster_name}/${local.cluster_name}-kubeconfig apply -f /root/bootstrap/ccm_secret.yaml", - "kubectl --kubeconfig /root/baremetal/bmctl-workspace/${local.cluster_name}/${local.cluster_name}-kubeconfig apply -f ${var.ccm_deploy_url}" + "kubectl --kubeconfig /root/baremetal/bmctl-workspace/${local.cluster_name}/${local.cluster_name}-kubeconfig apply -f ${local.ccm_deploy_url}" ] } } @@ -447,9 +421,6 @@ resource "null_resource" "worker_pre_reqs" { inline = ["mkdir -p /root/bootstrap/"] } - # Unless /root/bootstrap/ is created in advance, this will be - # copied to /root/bootstrap (file) - # https://github.com/hashicorp/terraform/issues/16330 provisioner "file" { content = data.template_file.pre_reqs_worker.rendered destination = "/root/bootstrap/pre_reqs_worker.sh" @@ -464,7 +435,7 @@ module "storage" { source = "./modules/storage" depends_on = [ - null_resource.add_kubelet_flags_to_workers, + null_resource.install_ccm, ] ssh = { diff --git a/modules/rook/variables.tf b/modules/rook/variables.tf index fc36f59..c9bfaec 100644 --- a/modules/rook/variables.tf +++ b/modules/rook/variables.tf @@ -6,8 +6,8 @@ variable "rook_version" { variable "latest_rook_version" { type = string - description = "The version of Rook to install" - default = "v1.5.10" + description = "The latest version of Rook that has been tested" + default = "v1.7.3" } variable "ssh" { diff --git a/modules/storage/variables.tf b/modules/storage/variables.tf index 024eedf..7e0f538 100644 --- a/modules/storage/variables.tf +++ b/modules/storage/variables.tf @@ -1,5 +1,5 @@ variable "storage_module" { - description = "The name of the Storage provider module (ex. \"portworx\")" + description = "The name of the Storage provider module (ex. \"rook\")" default = "" } diff --git a/templates/add_remaining_cps.sh b/templates/add_remaining_cps.sh index 68184f0..9053a59 100644 --- a/templates/add_remaining_cps.sh +++ b/templates/add_remaining_cps.sh @@ -1,12 +1,11 @@ #!/bin/bash CLUSTER_NAME='${cluster_name}' -CP_2='${cp_2}' -CP_3='${cp_3}' +CP_IP_2='${cp_ip_2}' +CP_ID_2='${cp_id_2}' +CP_IP_3='${cp_ip_3}' +CP_ID_3='${cp_id_3}' -# Wait a minute for things to settle -#echo "Waiting for 60 seconds to let the cluster settle" -#sleep 60 kubectl \ --kubeconfig /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME-kubeconfig \ -n cluster-$CLUSTER_NAME \ @@ -17,14 +16,16 @@ kubectl \ "op": "add", "path": "/spec/controlPlane/nodePoolSpec/nodes/1", "value": { - "address": "'$CP_2'" + "address": "'$CP_IP_2'", + "providerID": "equinixmetal://'$CP_ID_2'" } }, { "op": "add", "path": "/spec/controlPlane/nodePoolSpec/nodes/2", "value": { - "address": "'$CP_3'" + "address": "'$CP_IP_3'", + "providerID": "equinixmetal://'$CP_ID_3'" } } ]' diff --git a/templates/ccm_secret.yaml b/templates/ccm_secret.yaml index 409a49d..e23d971 100644 --- a/templates/ccm_secret.yaml +++ b/templates/ccm_secret.yaml @@ -1,3 +1,18 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: metal-cloud-config + namespace: kube-system +stringData: + cloud-sa.json: | + { + "apiKey": "${auth_token}", + "projectID": "${project_id}" + } + +# This is just to get Kube-Vip working again... We should fix this soon +--- apiVersion: v1 kind: Secret metadata: diff --git a/templates/kube_vip_install.sh b/templates/kube_vip_install.sh index ed1d35b..1960c00 100644 --- a/templates/kube_vip_install.sh +++ b/templates/kube_vip_install.sh @@ -69,9 +69,6 @@ wait_for_path "/root/bootstrap/vip.yaml" # Copy kube-vip manifest to the manifests folder cp /root/bootstrap/vip.yaml /etc/kubernetes/manifests/ -sed -i '/KUBELET_KUBEADM_ARGS/ s/"$/ --cloud-provider=external"/' /var/lib/kubelet/kubeadm-flags.env -sudo systemctl restart kubelet - if [[ "$COUNT" == "0" ]]; then printf "$${GREEN}BGP peering initiated! Cluster should be completed in about 5 minutes.$${NC}\n" fi diff --git a/templates/pre_reqs.sh b/templates/pre_reqs.sh index e47c42d..d774016 100644 --- a/templates/pre_reqs.sh +++ b/templates/pre_reqs.sh @@ -5,8 +5,9 @@ CP_VIP='${cp_vip}' INGRESS_VIP='${ingress_vip}' ANTHOS_VER='${anthos_ver}' IFS=' ' read -r -a CP_IPS <<< '${cp_ips}' +IFS=' ' read -r -a CP_IDS <<< '${cp_ids}' IFS=' ' read -r -a WORKER_IPS <<< '${worker_ips}' - +IFS=' ' read -r -a WORKER_IDS <<< '${worker_ids}' function ubuntu_pre_reqs { # Install Docker @@ -16,8 +17,8 @@ function ubuntu_pre_reqs { curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" -y sudo apt-get update -qy - DOCKER_VERSION=`sudo apt-cache madison docker-ce | grep '19.03.15' | awk '{print $3}'` - DOCKER_CLI_VERSION=`sudo apt-cache madison docker-ce-cli | grep '19.03.15' | awk '{print $3}'` + DOCKER_VERSION=`sudo apt-cache madison docker-ce | grep '19.03.13' | awk '{print $3}'` + DOCKER_CLI_VERSION=`sudo apt-cache madison docker-ce-cli | grep '19.03.13' | awk '{print $3}'` sudo apt-get install docker-ce=$DOCKER_VERSION docker-ce-cli=$DOCKER_CLI_VERSION -qy sudo usermod -aG docker $USER @@ -62,7 +63,7 @@ else fi # Install kubectl -curl -LO "https://storage.googleapis.com/kubernetes-release/release/v1.18.6/bin/linux/amd64/kubectl" +curl -LO "https://storage.googleapis.com/kubernetes-release/release/v1.20.9/bin/linux/amd64/kubectl" chmod a+x kubectl sudo mv kubectl /usr/local/bin/ @@ -80,9 +81,10 @@ chmod a+x bmctl bmctl_workspace='/root/baremetal/bmctl-workspace' cluster_config="$bmctl_workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml" GCP_PROJECT_ID=`grep 'project_id' /root/baremetal/keys/register.json | awk -F'"' '{print $4}'` -cp_string=" - address: $${CP_IPS[0]}"$'\\n' -for i in "$${WORKER_IPS[@]}"; do - worker_string="$worker_string - address: $i"$'\\n' +cp_string=" - address: $${CP_IPS[0]}"$'\\n'" providerID: equinixmetal://$${CP_IDS[0]}"$'\\n' +worker_count="$(($${#WORKER_IPS[@]}-1))" +for i in $(seq 0 $worker_count); do + worker_string="$worker_string - address: $${WORKER_IPS[$i]}"$'\\n'" providerID: equinixmetal://$${WORKER_IDS[$i]}"$'\\n' done # Replace variables in cluster config @@ -92,6 +94,7 @@ sed -i "s||/root/.ssh/id_rsa|g" $ sed -i "s||/root/baremetal/keys/connect.json|g" $cluster_config sed -i "s||/root/baremetal/keys/register.json|g" $cluster_config sed -i "s||/root/baremetal/keys/cloud-ops.json|g" $cluster_config +sed -i "s|name: $CLUSTER_NAME|name: $CLUSTER_NAME\\n annotations:\\n baremetal.cluster.gke.io/external-cloud-provider: \"true\"|g" $cluster_config sed -i "s|type: admin|type: hybrid|g" $cluster_config sed -i "s||$GCP_PROJECT_ID|g" $cluster_config sed -i "s| - address: ||g" $cluster_config @@ -101,5 +104,5 @@ sed -i "s|controlPlaneVIP: 10.0.0.8|controlPlaneVIP: $CP_VIP|g" $cluster_config sed -i "s|# ingressVIP: 10.0.0.2|ingressVIP: $INGRESS_VIP|g" $cluster_config sed -i "s| - address: |$cp_string|g" $cluster_config sed -i "s| - address: |$worker_string|g" $cluster_config -sed -i "s|- 10.96.0.0/12|- 172.31.0.0/16|g" $cluster_config +sed -i "s|- 10.96.0.0/20|- 172.31.0.0/16|g" $cluster_config sed -i "s|- 192.168.0.0/16|- 172.30.0.0/16|g" $cluster_config diff --git a/templates/worker_kubelet_flags.sh b/templates/worker_kubelet_flags.sh deleted file mode 100644 index fd1a607..0000000 --- a/templates/worker_kubelet_flags.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -GREEN='\033[0;32m' # Color green -YELLOW='\033[0;33m' # Color yellow -NC='\033[0m' # No Color - -function wait_for_path() { - if [[ $2 == 'dir' ]]; then - while [ ! -d $1 ]; do - printf "$${YELLOW}Waiting for '$1' to be created...$${NC}\n" - sleep 10 - done - else - while [ ! -f $1 ]; do - printf "$${YELLOW}Waiting for '$1' to be created...$${NC}\n" - sleep 10 - done - fi - printf "$${GREEN}$1 FOUND!$${NC}\n" -} - -wait_for_path "/var/lib/kubelet/kubeadm-flags.env" - -sed -i '/KUBELET_KUBEADM_ARGS/ s/"$/ --cloud-provider=external"/' /var/lib/kubelet/kubeadm-flags.env -sudo systemctl restart kubelet - diff --git a/variables.tf b/variables.tf index 2bd5905..b79fc46 100644 --- a/variables.tf +++ b/variables.tf @@ -106,7 +106,7 @@ variable "bgp_asn" { variable "ccm_version" { type = string - default = "v2.0.0" + default = "v3.2.2" description = "The version of the Equinix Metal CCM" } @@ -118,19 +118,13 @@ variable "kube_vip_version" { variable "anthos_version" { type = string - default = "1.7.0" + default = "1.8.3" description = "The version of Google Anthos to install" } -variable "ccm_deploy_url" { - type = string - default = "https://gist.githubusercontent.com/thebsdbox/c86dd970549638105af8d96439175a59/raw/4abf90fb7929ded3f7a201818efbb6164b7081f0/ccm.yaml" - description = "The deploy url for the Equinix Metal CCM" -} - variable "kube_vip_daemonset_url" { type = string - default = "https://raw.githubusercontent.com/plunder-app/kube-vip/bb7d2da73eeb6c4712479b007ff931a12180e626/docs/manifests/kube-vip-em.yaml" + default = "https://raw.githubusercontent.com/kube-vip/kube-vip/v0.3.8/docs/manifests/kube-vip-em.yaml" description = "The deploy url for the Kube-VIP Daemonset" } From dcc5798a34bfba535b16a1beea380c946f0424da Mon Sep 17 00:00:00 2001 From: Cody Hill Date: Thu, 16 Sep 2021 19:03:27 +0000 Subject: [PATCH 2/3] Fixed some minor formatting --- gcp-service-accts.tf | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gcp-service-accts.tf b/gcp-service-accts.tf index 908fab9..8ecd518 100644 --- a/gcp-service-accts.tf +++ b/gcp-service-accts.tf @@ -59,9 +59,8 @@ resource "google_project_iam_member" "cloud_ops_sa_role_metricwriter" { } resource "google_project_iam_member" "cloud_ops_sa_role_resourcewriter" { - count = local.sa_count - role = "roles/stackdriver.resourceMetadata.writer" - + count = local.sa_count + role = "roles/stackdriver.resourceMetadata.writer" member = format("%s:%s", local.sa_text, google_service_account.cloud_ops_sa[count.index].email) } From 7a79b76c8f06a2218ff68acf7b282cca90bb3d76 Mon Sep 17 00:00:00 2001 From: Marques Johansson Date: Wed, 29 Sep 2021 08:54:59 -0400 Subject: [PATCH 3/3] update kube-vip to 0.3.8 Signed-off-by: Marques Johansson --- .terraform.lock.hcl | 36 ++++++++++++++++++++--------------- README.md | 3 +-- gcp-service-accts.tf | 4 ++-- main.tf | 3 +++ templates/ccm_secret.yaml | 6 ++++-- templates/kube_vip_ds.yaml | 2 +- templates/kube_vip_install.sh | 16 +++++++--------- variables.tf | 8 +------- 8 files changed, 40 insertions(+), 38 deletions(-) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl index 88d2699..5a2e1db 100644 --- a/.terraform.lock.hcl +++ b/.terraform.lock.hcl @@ -2,22 +2,22 @@ # Manual edits may be lost in future updates. provider "registry.terraform.io/equinix/metal" { - version = "3.1.0" + version = "3.2.0" hashes = [ - "h1:IuYQx4pH1ooN7gt689g/nIy56hEkIop/9aVa+TXulv8=", - "zh:070b77456a3de02b623be29f1752b48c17100533a98661b7c6555ad2b60aeb1b", - "zh:140b4bfb72810b2e977e7fe442a5b69096db93fe3daff6e307574a541079eb42", - "zh:1d9bf5ba1e05b55af6f5d3e37fa9c0dd4c9fed02e0d47112945c16561f10fe38", - "zh:2b380fb077bb21296e1560c37efd051a7bb31fa488fbe525f54a166ace8cccc8", - "zh:5120fd9f3164e06473d813206e4f06d3b330a351ef6f59114546e0170a49f542", - "zh:5b131619662a36af7f0691c9505d24462533965391039b5664855e50b64db7c4", - "zh:7d3b2f2e2d1a0073ae54070e5dee37713726f6401f26d046db8a5cf58c80cebc", - "zh:7d40ec64e8a3f77c272c9e0cfc8bb6fc148e11a130ef0f022291947313798ec1", - "zh:7fe35422891a14ed233080303468221d8a9f98cd74b0fe73688569cbfc19ff4c", - "zh:a9bb9bfe0928bbc15c132004b3ef573983bb14cc59af9fb492b4f59f2d159054", - "zh:bd7000a809e2e96f87ac7ef90c953a05a8813feb220f27cccc50232f1ae02a50", - "zh:c0656d79d71b893fabcc8420774b8c2d5e2c77feb2b7fecd5dfb9be24228d392", - "zh:f1c755853d4c988fc616e8b1cf8407ed425a0e9ef35a441c0e9eb37657aa9907", + "h1:PIotyZNp3/sYe/Zk7yEgHygV5GOtHjAqXr4TJYdtETs=", + "zh:4a53b56d7cab5a75dca16c4a285438f13fe07a37c45967668dae8e4a70b78a85", + "zh:4fa88d43bd2370e420de6238f02f8ad668a0cd9e7f010a24a8bc0a76982ff3bc", + "zh:525b0f6d977437388b7ee7428bccd351e46af39a0883ce08f624b4dd173de8c2", + "zh:5c906ee75fbd1daa63dc40221015bfdcea7cec24ed7aa3687fcc18265556e379", + "zh:6153771a66d1f5058e977a686e706f3ced64cbcb7cd623a9adf7e8ed3f0d08d8", + "zh:79dd26afcc0846864ede59e79ae07fdf6cc084315cd9e50f4e09c700d9d8dfb4", + "zh:992e410067995537565f38ad9f93f4d66fabdc25a8303a1be418aefaf172daae", + "zh:9e52d2d008846cb6db8d0cd0aac7e1c6c635fc65c192b0217b2e673d165def0f", + "zh:c2ca5c36358c03c996d97b8baca6809675f54010c021d5704124bd3815660c86", + "zh:e2d4bfae4489142cfac31fb7939047226335c0fddf59c3544faeb1f3029a36d8", + "zh:e5c6727b6404beb3aa54bbc7eb3145bac29bab8135771f21754e093c178ee23f", + "zh:ecf34c7c8b3eb8cd61b381d8f26ec61b3d8b5d57a87fff97cb24331230286fef", + "zh:fcae7646861ccf3393828ac53d05487ba8547cf9f8a9da13553dcedf8dd2646e", ] } @@ -26,6 +26,7 @@ provider "registry.terraform.io/hashicorp/google" { constraints = "~> 3.53.0" hashes = [ "h1:0MYwK1KRNCc9lfF8vV9gDEuaylwEfSPws7ZJbLwY2FE=", + "h1:AzrT8ueZHo7GrEWFiXi3eB/NOQoXcVE/w6fLcRJyc34=", "zh:1408365b5f2ae508fce9b446bb9dbaf044aec81fa4c36fff39c2511b179bcc56", "zh:1d53e978065feb6278bc8c88a70c3df7599c3b8bbcd77765bcd842a83bce6686", "zh:5173a92249c8d06d0d2beca0e328df6e956becd789ebae9a064f022151415b8f", @@ -43,6 +44,7 @@ provider "registry.terraform.io/hashicorp/local" { version = "2.1.0" hashes = [ "h1:EYZdckuGU3n6APs97nS2LxZm3dDtGqyM4qaIvsmac8o=", + "h1:KfieWtVyGWwplSoLIB5usKAUnrIkDQBkWaR5TI+4WYg=", "zh:0f1ec65101fa35050978d483d6e8916664b7556800348456ff3d09454ac1eae2", "zh:36e42ac19f5d68467aacf07e6adcf83c7486f2e5b5f4339e9671f68525fc87ab", "zh:6db9db2a1819e77b1642ec3b5e95042b202aee8151a0256d289f2e141bf3ceb3", @@ -61,6 +63,7 @@ provider "registry.terraform.io/hashicorp/null" { version = "3.1.0" hashes = [ "h1:vpC6bgUQoJ0znqIKVFevOdq+YQw42bRq0u+H3nto8nA=", + "h1:xhbHC6in3nQryvTQBWKxebi3inG5OCgHgc4fRxL0ymc=", "zh:02a1675fd8de126a00460942aaae242e65ca3380b5bb192e8773ef3da9073fd2", "zh:53e30545ff8926a8e30ad30648991ca8b93b6fa496272cd23b26763c8ee84515", "zh:5f9200bf708913621d0f6514179d89700e9aa3097c77dac730e8ba6e5901d521", @@ -79,6 +82,7 @@ provider "registry.terraform.io/hashicorp/random" { version = "3.1.0" hashes = [ "h1:BZMEPucF+pbu9gsPk0G0BHx7YP04+tKdq2MrRDF1EDM=", + "h1:rKYu5ZUbXwrLG1w81k7H3nce/Ys6yAxXhWcbtk36HjY=", "zh:2bbb3339f0643b5daa07480ef4397bd23a79963cc364cdfbb4e86354cb7725bc", "zh:3cd456047805bf639fbf2c761b1848880ea703a054f76db51852008b11008626", "zh:4f251b0eda5bb5e3dc26ea4400dba200018213654b69b4a5f96abee815b4f5ff", @@ -96,6 +100,7 @@ provider "registry.terraform.io/hashicorp/random" { provider "registry.terraform.io/hashicorp/template" { version = "2.2.0" hashes = [ + "h1:0wlehNaxBX7GJQnPfQwTNvvAf38Jm0Nv7ssKGMaG6Og=", "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", @@ -113,6 +118,7 @@ provider "registry.terraform.io/hashicorp/template" { provider "registry.terraform.io/hashicorp/tls" { version = "3.1.0" hashes = [ + "h1:XTU9f6sGMZHOT8r/+LWCz2BZOPH127FBTPjMMEAAu1U=", "h1:fUJX8Zxx38e2kBln+zWr1Tl41X+OuiE++REjrEyiOM4=", "zh:3d46616b41fea215566f4a957b6d3a1aa43f1f75c26776d72a98bdba79439db6", "zh:623a203817a6dafa86f1b4141b645159e07ec418c82fe40acd4d2a27543cbaa2", diff --git a/README.md b/README.md index f64a6e8..4553be9 100644 --- a/README.md +++ b/README.md @@ -182,10 +182,9 @@ A complete list of variables can be found at