From 64bd9090bd441ba195e61a5002ad2aa930bdf834 Mon Sep 17 00:00:00 2001 From: Cody Hill <15899666+c0dyhi11@users.noreply.github.com> Date: Tue, 15 Dec 2020 13:59:46 -0600 Subject: [PATCH] Stable Production Grade Release (#23) This release incorporates the GA release of Anthos on Baremetal. It also included full BGP support from KubeVIP and has been tested on Ubuntu 18.04, Ubuntu 20.04, & CentOS 8. It has also been tested with an HA Control Plane (3x Control Plane nodes) As well as a single Control plane. You can deploy with as few as two nodes (1x Control Plane and 1x Worker Node) --- .gitignore | 3 + README.md | 276 +++++++++++++------------ main.tf | 325 +++++++++++++++++++++++------- output.tf | 37 ++-- templates/add_remaining_cps.sh | 33 +++ templates/ccm_secret.yaml | 11 + templates/create_cluster.sh | 10 + templates/kube_vip_ds.yaml | 91 +++++++++ templates/kube_vip_install.sh | 75 +++++++ templates/pre_reqs.sh | 71 ++++--- templates/update_cluster_vars.py | 36 ---- templates/user_data.sh | 53 +---- templates/worker_kubelet_flags.sh | 26 +++ util/setup_gcp_project.sh | 7 + variables.tf | 72 ++++++- 15 files changed, 792 insertions(+), 334 deletions(-) create mode 100644 templates/add_remaining_cps.sh create mode 100644 templates/ccm_secret.yaml create mode 100644 templates/create_cluster.sh create mode 100644 templates/kube_vip_ds.yaml create mode 100644 templates/kube_vip_install.sh delete mode 100644 templates/update_cluster_vars.py create mode 100644 templates/worker_kubelet_flags.sh diff --git a/.gitignore b/.gitignore index ed792fd..e5e9743 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Local .terraform directories **/.terraform/* +.terraform* # .tfstate files *.tfstate @@ -29,3 +30,5 @@ override.tf.json # example: *tfplan* terraform.tfvars util/keys + +*-kubeconfig diff --git a/README.md b/README.md index 0ec255e..c07c665 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,49 @@ -# Anthos Baremetal on Equinix Metal +[![Equinix Metal Website](https://img.shields.io/badge/Website%3A-metal.equinix.com-blue)](http://metal.equinix.com) [![Slack Status](https://slack.equinixmetal.com/badge.svg)](https://slack.equinixmetal.com/) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com) ![](https://img.shields.io/badge/Stability-Experimental-red.svg) -This is a very basic terraform template that will deploy **_N_** number of nodes (Defaults to 2) and configure a private vLan and IPs between these nodes using Equinix Metal's **_Mixed/Hybrid_** networking. +# Automated Anthos on Baremetal via Terraform for Equinix Metal +These files will allow you to use [Terraform](http://terraform.io) to deploy [Google Cloud's Anthos on Baremetal](https://cloud.google.com/anthos) on [Equinix Metal's Bare Metal Cloud offering](http://metal.equinix.com). -## Download this project +Terraform will create an Equinix Metal project complete with Linux machines for your Anthos on Baremetal cluster registered to Google Cloud. You can use an existing Equinix Metal Project, check this [section](#use-an-existing-equinix-metal-project) for instructions. -To download this project, run the following command: +![Environment Diagram](docs/images/google-anthos-vsphere-network-diagram-1.png) -```bash -git clone https://github.com/c0dyhi11/baremetal-anthos.git -cd baremetal-anthos -``` +Users are responsible for providing their Equinix Metal account, and Anthos subscription as described in this readme. + +The build (with default settings) typically takes 25-30 minutes. + +**The automation in the repo is COMMUNITY SUPPORTED ONLY**, if the installation succeeds, and you run the Anthos Platform Validation this cluster is production grade and supportable by Google for Anthos and packet for Infrastructure. If you have any questions please consult with Equinix Metal Support via a support ticket. + +## Join us on Slack +We use [Slack](https://slack.com/) as our primary communication tool for collaboration. You can join the Equinix Metal Community Slack group by going to [slack.equinixmetal.com](https://slack.equinixmetal.com/) and submitting your email address. You will receive a message with an invite link. Once you enter the Slack group, join the **#google-anthos** channel! Feel free to introduce yourself there, but know it's not mandatory. + +## Latest Updates +### 12-14-2020 +This is the initial release of this project. We support Ubuntu 20.04, Ubuntu 18.04, & CentOS 8. Red Hat Enterprise Linux is in the works. + +## Prerequisites +To use these Terraform files, you need to have the following Prerequisites: +* An [Anthos subscription](https://cloud.google.com/anthos/docs/getting-started) +* Google Cloud service-account keys, check this [section](#-service-account-generation) +* A Equinix Metal org-id and [API key](https://www.packet.com/developers/api/) + + +## Associated Equinix Metal Costs +The default variables make use of 6 [c3.small.x86](https://metal.equinix.com/product/servers/) servers. These servers are $0.50 per hour list price (resulting in a total solution price of roughly $3.00 per hour). This deployment has been test with as little as 2 [c3.small.x86](https://metal.equinix.com/product/servers/) (1 Control Plane node and 1 Worker node) for a total cost of roughly $1.00. + + +## Tested Anthos on Baremetal versions +The Terraform has been successfully tested with following versions of Anthos on Baremetal: +* 1.6.0 -## Generate your GCP Keys +To simplify setup, this is designed to use manual LoadBalancing with [Kube-VIP](https://kube-vip.io) load balancer. No other load balancer support is planned at this time. -There is a helper script in the `util` directory named `setup_gcp_project.sh`. You will need. The Google Cloud SDK installed to run this, and you'll most likley need to have the `Project Owner Role`. +Select the version of Anthos you wish to install by setting the `anthos_version` variable in your terraform.tfvars file. -Once your keys are generated you should have a folder named `keys` in the `util` directory with the following files: +## Download/Create your GCP Keys for your service accounts and activate APIs for your project +The Anthos on Baremetal install requires several service accounts and keys to be created. See the [Google documentation](https://cloud.google.com/anthos/gke/docs/bare-metal/1.6/installing/install-prereq#service_accounts_prerequisites) for more details. You can create these keys manually, or use a provided helper script to make the keys for you (Recommended). + +The Terraform files expect the keys to use the following naming convention, matching that of the Google documentation: ``` util |_keys @@ -27,6 +54,20 @@ util |_super-admin.json ``` +If doing so manually, you must create each of these keys and place it in a folder named `keys` within the `util` folder. +The service accounts also need to have IAM roles assigned to each of them. To do this manually, you'll need to follow the [instructions from Google](https://cloud.google.com/anthos/gke/docs/bare-metal/1.6/installing/install-prereq#service_accounts_prerequisites) + + +Anthos on Baremetal also requires [several APIs](https://cloud.google.com/gke-on-prem/docs/how-to/gcp-project#enable_apis) to be activated on your target project. + +Much easier (and recommended) is to use the helper script located in the `util` directory called `setup_gcp_project.sh` to create these keys, assign the IAM roles, and activate the APIs. The script will allow you to log into GCP with your user account and the GCP project for your Anthos cluster. + +You can run this script as follows: + +`util/setup_gcp_project.sh` + +Prompts will guide you through the setup. + ## Install Terraform Terraform is just a single binary. Visit their [download page](https://www.terraform.io/downloads.html), choose your operating system, make the binary executable, and move it into your path. @@ -34,155 +75,136 @@ Terraform is just a single binary. Visit their [download page](https://www.terra Here is an example for **macOS**: ```bash -curl -LO https://releases.hashicorp.com/terraform/0.13.5/terraform_0.13.5_darwin_amd64.zip -unzip terraform_0.13.5_darwin_amd64.zip +curl -LO https://releases.hashicorp.com/terraform/0.14.2/terraform_0.14.2_darwin_amd64.zip +unzip terraform_0.14.2_darwin_amd64.zip chmod +x terraform sudo mv terraform /usr/local/bin/ -rm -f terraform_0.13.5_darwin_amd64.zip +rm -f terraform_0.14.2_darwin_amd64.zip ``` Here is an example for **Linux**: ```bash -curl -LO https://releases.hashicorp.com/terraform/0.13.5/terraform_0.13.5_linux_amd64.zip -unzip terraform_0.13.5_linux_amd64.zip +curl -LO https://releases.hashicorp.com/terraform/0.14.2/terraform_0.14.2_linux_amd64.zip +unzip terraform_0.14.2_linux_amd64.zip chmod +x terraform sudo mv terraform /usr/local/bin/ -rm -f terraform_0.13.5_linux_amd64.zip +rm -f terraform_0.14.2_linux_amd64.zip ``` -## Modify your variables - -You will need to set two variables at a minimum and there are a lot more you may wish to modify in `variables.tf`. This file must be created in your `baremetal-anthos` directory + +## Download this project +To download this project, run the following command: ```bash -cat <terraform.tfvars -auth_token = "" -project_id = "" -EOF +git clone https://github.com/c0dyhi11/terraform-metal-anthos-on-baremetal.git +cd terraform-metal-anthos-on-baremetal ``` -There are a lot more variables that can be modified in the `variables.tf` file. Those variables are documented below. - -## Initialize Terraform - -Terraform uses modules to deploy infrastructure. In order to initialize the modules your simply run: `terraform init`. This should download modules into a hidden directory `.terraform` - -## Deploy terraform template - -```bash -terraform apply --auto-approve +## Initialize Terraform +Terraform uses modules to deploy infrastructure. In order to initialize the modules your simply run: +``` +terraform init ``` +This should download six modules into a hidden directory `.terraform` + +## Modify your variables +There are many variables which can be set to customize your install within `variables.tf`. The default variables to bring up a 6 node Anthos cluster with an HA Control Plane and three worker nodes using Equinix Metal's [c3.small.x86](https://metal.equinix.com/product/servers/). Change each default variable at your own risk. + +There are some variables you must set with a terraform.tfvars files. You need to set `auth_token` & `organization_id` to connect to Equinix Metal and the `project_name` which will be created in Equinix Metal. And it's a good idea to set `cluster_name` to identify your cluster in the GCP portal. + +The Anthos variables include `anthos_version` and `anthos_user_cluster_name`. + +Here is a quick command plus sample values to start file for you (make sure you adjust the variables to match your environment): +```bash +cat <terraform.tfvars +auth_token = "cefa5c94-e8ee-4577-bff8-1d1edca93ed8" +organization_id = "42259e34-d300-48b3-b3e1-d5165cd14169" +project_name = "anthos-packet-project-1" +cluster_name = "my-first-anthos-cluster" +EOF +``` +### Available Variables +| Variable Name | Type | Default Value | Description | +| :--------------------: | :-----: | :-------------------------: | :------------------------------------------------------ | +| auth_token | string | n/a | Equinix Metal API Key | +| project_id | string | n/a | Equinix Metal Project ID | +| organization_id | string | n/a. | Equinix Metal Organization ID | +| hostname | string | anthos-baremetal | The hostname for nodes | +| facility | string | sv15 | Equinix Metal Facility to deploy into | +| cp_plan | string | c3.small.x86 | Equinix Metal device type to deploy control plane nodes | +| worker_plan | string | c3.small.x86 | Equinix Metal device type to deploy for worker nodes | +| ha_control_plane | boolean | true | Do you want a highly available control plane? | +| worker_count | number | 3 | Number of baremetal worker nodes | +| operating_system | string | ubuntu_20_04 | The Operating system of the node | +| billing_cycle | string | hourly | How the node will be billed (Not usually changed) | +| cluster_name | string | equinix-metal-gke-cluster | The name of the GKE cluster | +| create_project | string | true | Create a new project for this deployment? | +| project_name | string | baremetal-anthos | The name of the project if 'create_project' is 'true'. | +| bgp_asn | string | 65000 | BGP ASN to peer with Equinix Metal | +| ccm_version | string | v2.0.0 | The version of the Equinix Metal CCM | +| kube_vip_version | string | 0.2.3 | The version of Kube-VIP to install | +| anthos_version | string | 1.6.0 | The version of Google Anthos to install | +| ccm_deploy_url | string | **Too Long to put here...** | The deploy url for the Equinix Metal CCM | +| kube_vip_daemonset_url | string | **Too Long to put here...** | The deploy url for the Kube-VIP Daemonset | + +#### Supported Operating Systems +| Name | Api Slug | +| :------------------------: | :----------: | +| CentOS 8 | centos_8 | +| Ubuntu 18.04 | ubuntu_18_04 | +| Ubutnu 20.04 | ubuntu_20_04 | + +##### Comming Soon +| Name | Api Slug | +| :-----------------------: | :----------: | +| Red Hat Enterprise Linux 8 | rhel_8 | -Once this is complete you should get output similar to this: -```console -Apply complete! Resources: 27 added, 0 changed, 0 destroyed. +## Deploy the Anthos on Baremetal cluster onto Equinix Metal + +All there is left to do now is to deploy the cluster: +```bash +terraform apply --auto-approve +``` +This should end with output similar to this: +``` +Apply complete! Resources: 28 added, 0 changed, 0 destroyed. Outputs: Control_Plane_Public_IPs = [ - "145.40.65.89", - "145.40.65.95", - "145.40.65.123", -] -Control_Plane_Hostnames = [ - "hostname-1x2yz-cp-01", - "hostname-1x2yz-cp-02", - "hostname-1x2yz-cp-03", -] -Control_Plane_Tags = [ - [ - "anthos", - "baremetal", - "172.29.254.2", - ], - [ - "anthos", - "baremetal", - "172.29.254.3", - ], - [ - "anthos", - "baremetal", - "172.29.254.4", - ], -] -Worker_Node_Hostnames = [ - "hostname-1x2yz-worker-01", - "hostname-1x2yz-worker-02", - "hostname-1x2yz-worker-03", -] -Worker_Node_Tags = [ - [ - "anthos", - "baremetal", - "172.29.254.5", - ], - [ - "anthos", - "baremetal", - "172.29.254.6", - ], - [ - "anthos", - "baremetal", - "172.29.254.7", - ], + "136.144.50.115", + "136.144.50.117", + "136.144.50.119", ] +Control_Plane_VIP = "145.40.65.107" +Ingress_VIP = "145.40.65.106" +Kubeconfig_location = "/home/cloud-user/git/baremetal-anthos/equinix-metal-gke-cluster-vomqb-kubeconfig" Worker_Public_IPs = [ - "145.40.65.103", - "145.40.65.61", - "145.40.65.155", + "136.144.50.123", + "145.40.64.221", + "136.144.50.105", ] +ssh_key_location = "/home/cloud-user/.ssh/bm-cluster-20201211211054" ``` -## Tested Operating Systems - -| Name | Api Slug | -| :-----------------------: | :----------: | -| CentOS 8 | centos_8 | -| Ubuntu 18.04 | ubuntu_18_04 | -| Ubutnu 20.04 | ubuntu_20_04 | - -## Operating Systems that need fixing -| Name | Api Slug | -| :-----------------------: | :----------: | -| Red Hat Enterprise Linux 8 | rhel_8 | - - -## Variables +You can see this output again at anytime by running `terraform output` -| Variable Name | Type | Default Value | Description | -| :--------------: | :-----: | :-----------------------: | :------------------------------------------------ | -| auth_token | string | n/a | Equinix Metal API Key | -| project_id | string | n/a | Equinix Metal Project ID | -| hostname | string | anthos-baremetal | The hostname for nodes | -| facility | string | sv15 | Equinix Metal Facility to deploy into | -| plan | string | c3.small.x86 | Equinix Metal device type to deploy | -| ha_control_plane | boolean | true | Do you want a highly available control plane? | -| worker_count | number | 3 | Number of worker nodes | -| operating_system | string | ubuntu_20_04 | The Operating system of the node | -| billing_cycle | string | hourly | How the node will be billed (Not usually changed) | -| private_subnet | string | 172.29.254.0/24 | Private IP Space to use for Layer2 | -| cluster_name | string | equinix-metal-gke-cluster | The name of teh GKE cluster | -## Testing script +## Use an existing Equinix Metal project +If you have an existing Equinix Metal project you can use it. +**YOU MUST ENABLE BGP PEERING ON YOUR PROJECT WITHOUT A PASSWORD** -I had the need to see if my terraform script would work on each operating system. So I wrote **_test_matrix.sh_** to handle this for me. This script is located in the `util` directory. This script has a bunch of operating systems in a list and will execute the terraform scripts into folder for each OS. The output of the terraform script is stored in that dedicated directory in a file called terraform.log. +Get your Project ID, navigate to the Project from the console.equinixmetal.com console and click on PROJECT SETTINGS, copy the PROJECT ID. -To run this script simply do the following: +add the following variables to your terraform.tfvars -```bash -./test_matrix.sh apply ``` - -Once you are finished, you can tear it all down as follows: - -```bash -./test_matrix.sh destroy +create_project = false +project_id = "YOUR-PROJECT-ID" ``` -## Know issues +## Google Anthos Documentation +Once Anthos is deployed on Equinix Metal, all of the documentation for using Google Anthos is located on the [Anthos Documentation Page](https://cloud.google.com/anthos/docs). -You can't create and destory the same `cluster_name` more than once without deleting the cluster from the Google Cloud GKE console. In order to try and remedy this. I'm adding a random 5-digit string to the `cluster_name` diff --git a/main.tf b/main.tf index 2122921..e13050f 100644 --- a/main.tf +++ b/main.tf @@ -2,25 +2,29 @@ provider "packet" { auth_token = var.auth_token } - resource "random_string" "cluster_suffix" { length = 5 special = false upper = false } +resource "packet_project" "new_project" { + count = var.create_project ? 1 : 0 + name = var.project_name + organization_id = var.organization_id + bgp_config { + deployment_type = "local" + asn = var.bgp_asn + } +} + locals { cp_count = var.ha_control_plane ? 3 : 1 cluster_name = format("%s-%s", var.cluster_name, random_string.cluster_suffix.result) timestamp = timestamp() timestamp_sanitized = replace(local.timestamp, "/[- TZ:]/", "") ssh_key_name = format("bm-cluster-%s", local.timestamp_sanitized) -} - -resource "packet_vlan" "private_vlan" { - facility = var.facility - project_id = var.project_id - description = "Private Network" + project_id = var.create_project ? packet_project.new_project[0].id : var.project_id } resource "tls_private_key" "ssh_key_pair" { @@ -39,23 +43,24 @@ resource "local_file" "cluster_private_key_pem" { file_permission = "0600" } -data "template_file" "control_plane_user_data" { - count = local.cp_count - template = file("templates/user_data.sh") - vars = { - operating_system = var.operating_system - ip_address = cidrhost(var.private_subnet, count.index + 1) - netmask = cidrnetmask(var.private_subnet) - } +resource "packet_reserved_ip_block" "cp_vip" { + project_id = local.project_id + facility = var.facility + quantity = 1 + description = format("Cluster: '%s' Contol Plane VIP", local.cluster_name) +} + +resource "packet_reserved_ip_block" "ingress_vip" { + project_id = local.project_id + facility = var.facility + quantity = 1 + description = format("Cluster: '%s' Ingress VIP", local.cluster_name) } -data "template_file" "worker_user_data" { - count = var.worker_count +data "template_file" "user_data" { template = file("templates/user_data.sh") vars = { operating_system = var.operating_system - ip_address = cidrhost(var.private_subnet, local.cp_count + count.index + 1) - netmask = cidrnetmask(var.private_subnet) } } @@ -69,9 +74,9 @@ resource "packet_device" "control_plane" { facilities = [var.facility] operating_system = var.operating_system billing_cycle = var.billing_cycle - project_id = var.project_id - tags = ["anthos", "baremetal", cidrhost(var.private_subnet, count.index + 1)] - user_data = element(data.template_file.control_plane_user_data.*.rendered, count.index) + project_id = local.project_id + user_data = data.template_file.user_data.rendered + tags = ["anthos", "baremetal", "control-plane"] } resource "packet_device" "worker_nodes" { @@ -84,35 +89,21 @@ resource "packet_device" "worker_nodes" { facilities = [var.facility] operating_system = var.operating_system billing_cycle = var.billing_cycle - project_id = var.project_id - tags = ["anthos", "baremetal", cidrhost(var.private_subnet, local.cp_count + count.index + 1)] - user_data = element(data.template_file.worker_user_data.*.rendered, count.index) + project_id = local.project_id + user_data = data.template_file.user_data.rendered + tags = ["anthos", "baremetal", "worker"] } -resource "packet_device_network_type" "control_plane" { - count = local.cp_count - device_id = element(packet_device.control_plane.*.id, count.index) - type = "hybrid" +resource "packet_bgp_session" "enable_cp_bgp" { + count = local.cp_count + device_id = element(packet_device.control_plane.*.id, count.index) + address_family = "ipv4" } -resource "packet_device_network_type" "worker_nodes" { - count = var.worker_count - device_id = element(packet_device.worker_nodes.*.id, count.index) - type = "hybrid" -} - -resource "packet_port_vlan_attachment" "control_plane_vlan_attach" { - count = local.cp_count - device_id = element(packet_device_network_type.control_plane.*.id, count.index) - port_name = "eth1" - vlan_vnid = packet_vlan.private_vlan.vxlan -} - -resource "packet_port_vlan_attachment" "worker_vlan_attach" { - count = var.worker_count - device_id = element(packet_device_network_type.worker_nodes.*.id, count.index) - port_name = "eth1" - vlan_vnid = packet_vlan.private_vlan.vxlan +resource "packet_bgp_session" "enable_worker_bgp" { + count = var.worker_count + device_id = element(packet_device.worker_nodes.*.id, count.index) + address_family = "ipv4" } resource "null_resource" "write_ssh_private_key" { @@ -137,16 +128,11 @@ data "template_file" "deploy_anthos_cluster" { vars = { cluster_name = local.cluster_name operating_system = var.operating_system - } -} - -data "template_file" "update_cluster_vars" { - template = file("templates/update_cluster_vars.py") - vars = { - private_subnet = var.private_subnet - cp_count = local.cp_count - worker_count = var.worker_count - cluster_name = local.cluster_name + cp_vip = cidrhost(packet_reserved_ip_block.cp_vip.cidr_notation, 0) + ingress_vip = cidrhost(packet_reserved_ip_block.ingress_vip.cidr_notation, 0) + cp_ips = join(" ", packet_device.control_plane.*.access_private_ipv4) + worker_ips = join(" ", packet_device.worker_nodes.*.access_private_ipv4) + anthos_ver = var.anthos_version } } @@ -159,7 +145,10 @@ resource "null_resource" "prep_anthos_cluster" { } provisioner "remote-exec" { - inline = ["mkdir -p /root/baremetal/keys/"] + inline = [ + "mkdir -p /root/baremetal/keys/", + "mkdir -p /root/bootstrap/" + ] } provisioner "file" { @@ -169,24 +158,218 @@ resource "null_resource" "prep_anthos_cluster" { provisioner "file" { content = data.template_file.deploy_anthos_cluster.rendered - destination = "/root/baremetal/pre_reqs.sh" + destination = "/root/bootstrap/pre_reqs.sh" + } + + provisioner "remote-exec" { + inline = ["bash /root/bootstrap/pre_reqs.sh"] + } +} + +data "template_file" "create_cluster" { + template = file("templates/create_cluster.sh") + vars = { + cluster_name = local.cluster_name + } +} + +resource "null_resource" "deploy_anthos_cluster" { + depends_on = [ + null_resource.prep_anthos_cluster, + null_resource.write_ssh_private_key + ] + connection { + type = "ssh" + user = "root" + private_key = chomp(tls_private_key.ssh_key_pair.private_key_pem) + host = packet_device.control_plane.0.access_public_ipv4 } provisioner "file" { - content = data.template_file.update_cluster_vars.rendered - destination = "/root/baremetal/update_cluster_vars.py" + content = data.template_file.create_cluster.rendered + destination = "/root/bootstrap/create_cluster.sh" } provisioner "remote-exec" { - inline = ["bash /root/baremetal/pre_reqs.sh"] + inline = [ + "bash /root/bootstrap/create_cluster.sh" + ] } } -resource "null_resource" "deploy_anthos_cluster" { +resource "null_resource" "download_kube_config" { + depends_on = [null_resource.deploy_anthos_cluster] + + provisioner "local-exec" { + command = "scp -i ~/.ssh/${local.ssh_key_name} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@${packet_device.control_plane.0.access_public_ipv4}:/root/baremetal/bmctl-workspace/${local.cluster_name}/${local.cluster_name}-kubeconfig ." + } +} + +data "template_file" "template_kube_vip_install" { + count = var.ha_control_plane ? 2 : 1 + template = file("templates/kube_vip_install.sh") + vars = { + cluster_name = local.cluster_name + eip = cidrhost(packet_reserved_ip_block.cp_vip.cidr_notation, 0) + count = count.index + kube_vip_ver = var.kube_vip_version + auth_token = var.auth_token + project_id = local.project_id + } +} + +resource "null_resource" "kube_vip_install_first_cp" { + depends_on = [ + packet_bgp_session.enable_cp_bgp, + packet_bgp_session.enable_worker_bgp, + null_resource.prep_anthos_cluster, + ] + connection { + type = "ssh" + user = "root" + private_key = chomp(tls_private_key.ssh_key_pair.private_key_pem) + host = packet_device.control_plane.0.access_public_ipv4 + } + provisioner "file" { + content = data.template_file.template_kube_vip_install.0.rendered + destination = "/root/bootstrap/kube_vip_install.sh" + } + provisioner "remote-exec" { + inline = [ + "bash /root/bootstrap/kube_vip_install.sh" + ] + } +} + + +data "template_file" "add_remaining_cps" { + count = var.ha_control_plane ? 1 : 0 + template = file("templates/add_remaining_cps.sh") + vars = { + cluster_name = local.cluster_name + cp_2 = packet_device.control_plane.1.access_private_ipv4 + cp_3 = packet_device.control_plane.2.access_private_ipv4 + } +} + +resource "null_resource" "add_remaining_cps" { + count = var.ha_control_plane ? 1 : 0 + depends_on = [ + null_resource.deploy_anthos_cluster, + null_resource.kube_vip_install_first_cp + ] + connection { + type = "ssh" + user = "root" + private_key = chomp(tls_private_key.ssh_key_pair.private_key_pem) + host = packet_device.control_plane.0.access_public_ipv4 + } + provisioner "file" { + content = data.template_file.add_remaining_cps.0.rendered + destination = "/root/bootstrap/add_remaining_cps.sh" + } + provisioner "remote-exec" { + inline = [ + "bash /root/bootstrap/add_remaining_cps.sh" + ] + } +} + +resource "null_resource" "kube_vip_install_remaining_cp" { + count = var.ha_control_plane ? 2 : 0 depends_on = [ - packet_port_vlan_attachment.control_plane_vlan_attach, - packet_port_vlan_attachment.worker_vlan_attach, - null_resource.prep_anthos_cluster + null_resource.add_remaining_cps + ] + connection { + type = "ssh" + user = "root" + private_key = chomp(tls_private_key.ssh_key_pair.private_key_pem) + host = element(packet_device.control_plane.*.access_public_ipv4, count.index + 1) + } + provisioner "remote-exec" { + inline = ["mkdir -p /root/bootstrap"] + } + provisioner "file" { + content = data.template_file.template_kube_vip_install.1.rendered + destination = "/root/bootstrap/kube_vip_install.sh" + } + provisioner "remote-exec" { + inline = [ + "bash /root/bootstrap/kube_vip_install.sh" + ] + } +} + +data "template_file" "worker_kubelet_flags" { + template = file("templates/worker_kubelet_flags.sh") +} + +resource "null_resource" "add_kubelet_flags_to_workers" { + count = var.worker_count + depends_on = [ + null_resource.kube_vip_install_remaining_cp, + null_resource.deploy_anthos_cluster, + null_resource.kube_vip_install_first_cp + ] + connection { + type = "ssh" + user = "root" + private_key = chomp(tls_private_key.ssh_key_pair.private_key_pem) + host = element(packet_device.worker_nodes.*.access_public_ipv4, count.index) + } + provisioner "remote-exec" { + inline = [ + "mkdir -p /root/bootstrap/" + ] + } + provisioner "file" { + content = data.template_file.worker_kubelet_flags.rendered + destination = "/root/bootstrap/worker_kubelet_flags.sh" + } + provisioner "remote-exec" { + inline = [ + "bash /root/bootstrap/worker_kubelet_flags.sh" + ] + } +} + +data "template_file" "ccm_secret" { + template = file("templates/ccm_secret.yaml") + vars = { + auth_token = var.auth_token + project_id = local.project_id + } +} + +resource "null_resource" "install_ccm" { + depends_on = [ + null_resource.add_kubelet_flags_to_workers + ] + connection { + type = "ssh" + user = "root" + private_key = chomp(tls_private_key.ssh_key_pair.private_key_pem) + host = packet_device.control_plane.0.access_public_ipv4 + } + provisioner "file" { + content = data.template_file.ccm_secret.rendered + destination = "/root/bootstrap/ccm_secret.yaml" + } + provisioner "remote-exec" { + inline = [ + "kubectl --kubeconfig /root/baremetal/bmctl-workspace/${local.cluster_name}/${local.cluster_name}-kubeconfig apply -f /root/bootstrap/ccm_secret.yaml", + "kubectl --kubeconfig /root/baremetal/bmctl-workspace/${local.cluster_name}/${local.cluster_name}-kubeconfig apply -f ${var.ccm_deploy_url}" + ] + } +} + +data "template_file" "kube_vip_ds" { + template = file("templates/kube_vip_ds.yaml") +} + +resource "null_resource" "install_kube_vip_daemonset" { + depends_on = [ + null_resource.install_ccm ] connection { type = "ssh" @@ -194,12 +377,14 @@ resource "null_resource" "deploy_anthos_cluster" { private_key = chomp(tls_private_key.ssh_key_pair.private_key_pem) host = packet_device.control_plane.0.access_public_ipv4 } + provisioner "file" { + content = data.template_file.kube_vip_ds.rendered + destination = "/root/bootstrap/kube_vip_ds.yaml" + } provisioner "remote-exec" { inline = [ - "python3 /root/baremetal/update_cluster_vars.py", - "cd /root/baremetal/", - "export GOOGLE_APPLICATION_CREDENTIALS=/root/baremetal/keys/super-admin.json", - "/root/baremetal/bmctl create cluster -c ${local.cluster_name} --force" + "kubectl --kubeconfig /root/baremetal/bmctl-workspace/${local.cluster_name}/${local.cluster_name}-kubeconfig apply -f /root/bootstrap/kube_vip_ds.yaml" ] } } + diff --git a/output.tf b/output.tf index 46bf9c1..63f6b82 100644 --- a/output.tf +++ b/output.tf @@ -1,13 +1,3 @@ -output "Control_Plane_Hostnames" { - value = packet_device.control_plane.*.hostname - description = "Control Plane Hostnames" -} - -output "Worker_Node_Hostnames" { - value = packet_device.worker_nodes.*.hostname - description = "Worker Node hostnames" -} - output "Control_Plane_Public_IPs" { value = packet_device.control_plane.*.access_public_ipv4 description = "Control Plane Public IPs" @@ -18,18 +8,27 @@ output "Worker_Public_IPs" { description = "Worker Node Public IPs" } -output "Control_Plane_Tags" { - value = packet_device.control_plane.*.tags - description = "Control Plane Tags" +output "ssh_key_location" { + value = local_file.cluster_private_key_pem.filename + description = "The SSH Private Key File Location" } -output "Worker_Node_Tags" { - value = packet_device.worker_nodes.*.tags - description = "Worker Node Tags" +output "Control_Plane_VIP" { + value = cidrhost(packet_reserved_ip_block.cp_vip.cidr_notation, 0) + description = "The Virtual IP for the Control Plane" } -output "ssh_key_location" { - value = local_file.cluster_private_key_pem.filename - description = "The SSH Private Key File Location" +output "Ingress_VIP" { + value = cidrhost(packet_reserved_ip_block.ingress_vip.cidr_notation, 0) + description = "The Virtual IP for Ingress" } +output "Kubeconfig_location" { + value = format("%s/%s-kubeconfig", abspath(path.root), local.cluster_name) + description = "The path to your kubeconfig" +} + +output "Equinix_Metal_Project_ID" { + value = local.project_id + description = "The project ID used for this deployment" +} diff --git a/templates/add_remaining_cps.sh b/templates/add_remaining_cps.sh new file mode 100644 index 0000000..d4ee1f2 --- /dev/null +++ b/templates/add_remaining_cps.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +CLUSTER_NAME='${cluster_name}' +CP_2='${cp_2}' +CP_3='${cp_3}' + +mkdir -p /root/.kube/ + +cp /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME-kubeconfig /root/.kube/config +# Wait a minute for things to settle +#echo "Waiting for 60 seconds to let the cluster settle" +#sleep 60 +kubectl \ + --kubeconfig /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME-kubeconfig \ + -n cluster-$CLUSTER_NAME \ + patch cluster $CLUSTER_NAME \ + --type json \ + -p '[ + { + "op": "add", + "path": "/spec/controlPlane/nodePoolSpec/nodes/1", + "value": { + "address": "'$CP_2'" + } + }, + { + "op": "add", + "path": "/spec/controlPlane/nodePoolSpec/nodes/2", + "value": { + "address": "'$CP_3'" + } + } + ]' diff --git a/templates/ccm_secret.yaml b/templates/ccm_secret.yaml new file mode 100644 index 0000000..409a49d --- /dev/null +++ b/templates/ccm_secret.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Secret +metadata: + name: packet-cloud-config + namespace: kube-system +stringData: + cloud-sa.json: | + { + "apiKey": "${auth_token}", + "projectID": "${project_id}" + } diff --git a/templates/create_cluster.sh b/templates/create_cluster.sh new file mode 100644 index 0000000..de996c1 --- /dev/null +++ b/templates/create_cluster.sh @@ -0,0 +1,10 @@ +#!/bin/bash +CLUSTER_NAME='${cluster_name}' + +cd /root/baremetal +export GOOGLE_APPLICATION_CREDENTIALS=/root/baremetal/keys/super-admin.json +GREEN='\033[0;32m' # Color green +NC='\033[0m' # No Color +printf "$${GREEN}Creating Anthos Cluster. This will take about 20 minutes...$${NC}\n" +/root/baremetal/bmctl create cluster -c $CLUSTER_NAME --force &> /root/baremetal/cluster_create.log +printf "$${GREEN}Cluster Created!$${NC}\n" diff --git a/templates/kube_vip_ds.yaml b/templates/kube_vip_ds.yaml new file mode 100644 index 0000000..9b20b89 --- /dev/null +++ b/templates/kube_vip_ds.yaml @@ -0,0 +1,91 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services", "services/status"] + verbs: ["list","get","watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: + name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + name: kube-vip-ds + spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "false" + - name: vip_interface + value: lo + - name: port + value: "6443" + - name: vip_cidr + value: "32" + - name: svc_enable + value: "true" + - name: provider_config + value: /etc/cloud-sa/cloud-sa.json + - name: vip_packet + value: "true" + - name: bgp_enable + value: "true" + image: plndr/kube-vip:0.2.3 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_TIME + volumeMounts: + - mountPath: /etc/cloud-sa + name: cloud-sa-volume + readOnly: true + hostNetwork: true + serviceAccountName: kube-vip + volumes: + - name: cloud-sa-volume + secret: + secretName: packet-cloud-config + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/templates/kube_vip_install.sh b/templates/kube_vip_install.sh new file mode 100644 index 0000000..035ec1d --- /dev/null +++ b/templates/kube_vip_install.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +export EIP='${eip}' +KUBE_VIP_VER='${kube_vip_ver}' +CLUSTER_NAME='${cluster_name}' +COUNT='${count}' +PACKET_API_KEY='${auth_token}' +PACKET_PROJECT_ID='${project_id}' +GREEN='\033[0;32m' # Color green +YELLOW='\033[0;33m' # Color green +NC='\033[0m' # No Color + +function wait_for_path() { + if [[ $2 == 'dir' ]]; then + while [ ! -d $1 ]; do + printf "$${YELLOW}Waiting for '$1' to be created...$${NC}\n" + sleep 10 + done + else + while [ ! -f $1 ]; do + printf "$${YELLOW}Waiting for '$1' to be created...$${NC}\n" + sleep 10 + done + fi + printf "$${GREEN}$1 FOUND!$${NC}\n" +} + +function gen_kube_vip () { + sudo docker run --network host --rm plndr/kube-vip:$KUBE_VIP_VER manifest pod \ + --interface lo \ + --vip $EIP \ + --port 6444 \ + --controlplane \ + --bgp \ + --packet \ + --packetKey $PACKET_API_KEY \ + --packetProjectID $PACKET_PROJECT_ID \ + | sudo tee /root/bootstrap/vip.yaml + # Hack until manifest doesn't include this path + sed -i "/\/etc\/ssl\/certs/,+2 d" /root/bootstrap/vip.yaml +} + +function wait_for_docker () { + + while ! docker ps &> /dev/null; do + printf "$${YELLOW}Docker not installed yet... Sleeping 10Sec!$${NC}\n" + sleep 10 + done + printf "$${GREEN}Docker installed!$${NC}\n" +} + +wait_for_docker +wait_for_path "/etc/kubernetes/manifests" "dir" +wait_for_path "/var/lib/kubelet/kubeadm-flags.env" +if [[ "$COUNT" == "0" ]]; then + wait_for_path "/etc/kubernetes/admin.conf" + wait_for_path "/root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME-kubeconfig" + gen_kube_vip +elif [[ "$COUNT" == "1" ]]; then + wait_for_path "/etc/kubernetes/admin.conf" + printf "$${YELLOW}Wait a full minute before adding Kube-VIP or the cluster join will not complete...$${NC}\n" + sleep 60 + gen_kube_vip +fi +wait_for_path "/root/bootstrap/vip.yaml" +# Copy kube-vip manifest to the manifests folder +cp /root/bootstrap/vip.yaml /etc/kubernetes/manifests/ + +sed -i '/KUBELET_KUBEADM_ARGS/ s/"$/ --cloud-provider=external"/' /var/lib/kubelet/kubeadm-flags.env +sudo systemctl restart kubelet + +if [[ "$COUNT" == "0" ]]; then + printf "$${GREEN}BGP peering initiated! Cluster should be completed in about 5 minutes.$${NC}\n" +fi + diff --git a/templates/pre_reqs.sh b/templates/pre_reqs.sh index 52bf9cd..2a006cb 100644 --- a/templates/pre_reqs.sh +++ b/templates/pre_reqs.sh @@ -1,22 +1,30 @@ #!/bin/bash CLUSTER_NAME='${cluster_name}' OS='${operating_system}' +CP_VIP='${cp_vip}' +INGRESS_VIP='${ingress_vip}' +ANTHOS_VER='${anthos_ver}' +IFS=' ' read -r -a CP_IPS <<< '${cp_ips}' +IFS=' ' read -r -a WORKER_IPS <<< '${worker_ips}' + function ubuntu_pre_reqs { # Install Docker - sudo apt update -y - sudo apt install apt-transport-https ca-certificates curl software-properties-common gnupg -y + export DEBIAN_FRONTEND=noninteractive + sudo apt update -qy + sudo apt install apt-transport-https ca-certificates curl software-properties-common gnupg -qy curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu focal stable" -y - sudo apt update -y - sudo apt install docker-ce -y + sudo apt update -qy + DOCKER_VERSION=`sudo apt-cache madison docker-ce | grep '19.03.13' | awk '{print $3}'` + sudo apt install docker-ce=$DOCKER_VERSION -qy sudo usermod -aG docker $USER # Install Google Cloud SDK echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - - sudo apt-get update -y - sudo apt-get install google-cloud-sdk -y + sudo apt-get update -qy + sudo apt-get install google-cloud-sdk -qy } @@ -36,10 +44,11 @@ enabled=1 gpgcheck=1 repo_gpgcheck=1 gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg - https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg + https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg EOM - - sudo dnf install docker-ce iptables google-cloud-sdk python3 -y + sudo yum -q makecache -y --disablerepo='*' --enablerepo=google-cloud-sdk + DOCKER_VERSION=`sudo dnf --showduplicates list docker-ce | grep '19.03.13' | awk '{print $2}'` + sudo dnf install docker-ce-$DOCKER_VERSION iptables google-cloud-sdk python3 -y sudo systemctl enable --now docker } @@ -61,26 +70,40 @@ curl -LO "https://storage.googleapis.com/kubernetes-release/release/v1.18.6/bin/ chmod a+x kubectl sudo mv kubectl /usr/local/bin/ +# Assing CP VIP to first master node's lo interface +export EIP=$CP_VIP +ip add add $EIP/32 dev lo + # Download bmctl cd /root/baremetal gcloud auth activate-service-account --key-file=keys/gcr.json -gsutil cp gs://anthos-baremetal-release/bmctl/0.7.0-gke.6/linux/bmctl . +gsutil cp gs://anthos-baremetal-release/bmctl/$ANTHOS_VER/linux-amd64/bmctl . chmod a+x bmctl ./bmctl create config -c $CLUSTER_NAME - -# Replace variables in cluster config -sed -i "/Node pool configuration is only valid for 'bundled' LB mode./,+4 d" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml -sed -i "s||/root/baremetal/keys/gcr.json|g" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml -sed -i "s||/root/.ssh/id_rsa|g" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml -sed -i "s||/root/baremetal/keys/connect.json|g" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml -sed -i "s||/root/baremetal/keys/register.json|g" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml -sed -i "s||/root/baremetal/keys/cluster-ops.json|g" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml +bmctl_workspace='/root/baremetal/bmctl-workspace' +cluster_config="$bmctl_workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml" GCP_PROJECT_ID=`grep 'project_id' /root/baremetal/keys/register.json | awk -F'"' '{print $4}'` -sed -i "s|type: admin|type: hybrid|g" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml -sed -i "s||$GCP_PROJECT_ID|g" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml -sed -i "s| - address: ||g" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml -sed -i "s| # addressPools:| addressPools:|g" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml -sed -i "s| # - name: pool1| - name: pool1|g" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml -sed -i "s| # addresses:| addresses:|g" /root/baremetal/bmctl-workspace/$CLUSTER_NAME/$CLUSTER_NAME.yaml +cp_string=" - address: $${CP_IPS[0]}"$'\\n' +for i in "$${WORKER_IPS[@]}"; do + worker_string="$worker_string - address: $i"$'\\n' +done +# Replace variables in cluster config +sed -i "/Node pool configuration is only valid for 'bundled' LB mode./,+4 d" $cluster_config +sed -i "s||/root/baremetal/keys/gcr.json|g" $cluster_config +sed -i "s||/root/.ssh/id_rsa|g" $cluster_config +sed -i "s||/root/baremetal/keys/connect.json|g" $cluster_config +sed -i "s||/root/baremetal/keys/register.json|g" $cluster_config +sed -i "s||/root/baremetal/keys/cluster-ops.json|g" $cluster_config +sed -i "s|type: admin|type: hybrid|g" $cluster_config +sed -i "s||$GCP_PROJECT_ID|g" $cluster_config +sed -i "s| - address: ||g" $cluster_config +sed -i "s|mode: bundled|mode: manual|g" $cluster_config +sed -i "s|controlPlaneLBPort: 443|controlPlaneLBPort: 6444|g" $cluster_config +sed -i "s|controlPlaneVIP: 10.0.0.8|controlPlaneVIP: $CP_VIP|g" $cluster_config +sed -i "s|# ingressVIP: 10.0.0.2|ingressVIP: $INGRESS_VIP|g" $cluster_config +sed -i "s| - address: |$cp_string|g" $cluster_config +sed -i "s| - address: |$worker_string|g" $cluster_config +sed -i "s|- 10.96.0.0/12|- 172.31.0.0/16|g" $cluster_config +sed -i "s|- 192.168.0.0/16|- 172.30.0.0/16|g" $cluster_config diff --git a/templates/update_cluster_vars.py b/templates/update_cluster_vars.py deleted file mode 100644 index f309877..0000000 --- a/templates/update_cluster_vars.py +++ /dev/null @@ -1,36 +0,0 @@ -import ipaddress -import os - - -# Terraform VARs -PRIV_SUBNET = '${private_subnet}' -NUM_CPS = ${cp_count} -NUM_WORKERS = ${worker_count} -CLUSTER_NAME = '${cluster_name}' - -num_total = NUM_CPS + NUM_WORKERS -cp_string = '' -worker_string = '' - -for i in range(0, NUM_CPS): - cp_string += " - address: {}\n".format(list(ipaddress.ip_network(PRIV_SUBNET).hosts())[i].compressed) - -for i in range(NUM_CPS, num_total): - worker_string += " - address: {}\n".format(list(ipaddress.ip_network(PRIV_SUBNET).hosts())[i].compressed) - -cluster_vip = list(ipaddress.ip_network(PRIV_SUBNET).hosts())[-1].compressed -ingress_vip = list(ipaddress.ip_network(PRIV_SUBNET).hosts())[-2].compressed -lb_vip_end = list(ipaddress.ip_network(PRIV_SUBNET).hosts())[-2].compressed -lb_vip_start = list(ipaddress.ip_network(PRIV_SUBNET).hosts())[-155].compressed -lb_vip_range = "{}-{}".format(lb_vip_start, lb_vip_end) - -file_path = "/root/baremetal/bmctl-workspace/{0}/{0}.yaml".format(CLUSTER_NAME) -# String Replacements -os.system("sed -i 's| - address: |{}|g' {}".format(cp_string, - file_path).encode("unicode_escape").decode("utf-8")) -os.system("sed -i 's| - address: |{}|g' {}".format(worker_string, - file_path).encode("unicode_escape").decode("utf-8")) -os.system("sed -i 's|controlPlaneVIP: 10.0.0.8|controlPlaneVIP: {}|g' {}".format(cluster_vip, file_path)) -os.system("sed -i 's|# ingressVIP: 10.0.0.2|ingressVIP: {}|g' {}".format(ingress_vip, file_path)) -os.system("sed -i 's|# - 10.0.0.1-10.0.0.4| - {}|g' {}".format(lb_vip_range, file_path)) - diff --git a/templates/user_data.sh b/templates/user_data.sh index ff6c0e8..5d5f321 100644 --- a/templates/user_data.sh +++ b/templates/user_data.sh @@ -1,54 +1,11 @@ #!/bin/bash OS='${operating_system}' -IP_ADDRESS='${ip_address}' -NETMASK='${netmask}' - - -function rhel_config { - nic=`ip a | grep "master bond0" | tail -1 | awk '{print $2}' | awk -F':' '{print $1}'` - ifdown $nic - cat <<-EOF > /etc/sysconfig/network-scripts/ifcfg-$nic - DEVICE=$nic - ONBOOT=yes - BOOTPROTO=none - IPADDR=$IP_ADDRESS - NETMASK=$NETMASK -EOF - ifup $nic -} - - -function ubuntu_config { - nic=`grep auto /etc/network/interfaces | tail -1 | awk '{print $2}'` - ifdown $nic - head -n -5 /etc/network/interfaces > /etc/network/interfaces - printf "\nauto $nic\n" >> /etc/network/interfaces - printf "iface $nic inet static\n" >> /etc/network/interfaces - printf "\taddress $IP_ADDRESS\n" >> /etc/network/interfaces - printf "\tnetmask $NETMASK\n" >> /etc/network/interfaces - ifup $nic -} - - -function unknown_config { - echo "I don't konw who I am" > /root/who_am_i.txt -} - -function sync_time { - systemctl stop ntp.service - ntpd -gq - systemctl start ntp.service - timedatectl > /tmp/timedatctl.txt -} - - -sync_time if [ "$${OS:0:6}" = "centos" ] || [ "$${OS:0:4}" = "rhel" ]; then - rhel_config -elif [ "$${OS:0:6}" = "ubuntu" ]; then - ubuntu_config -else - unknown_config + dnf install jq -y fi +GATEWAY_IP=$(curl https://metadata.platformequinix.com/metadata | jq -r ".network.addresses[] | select(.public == false) | .gateway") +sed -i.bak -E "/^\s+post-down route del -net 10\.0\.0\.0.* gw .*$/a \ \ \ \ up ip route add 169.254.255.1 via $GATEWAY_IP || true\n up ip route add 169.254.255.2 via $GATEWAY_IP || true\n down ip route del 169.254.255.1 || true\n down ip route del 169.254.255.2 || true" /etc/network/interfaces +ip route add 169.254.255.1 via $GATEWAY_IP +ip route add 169.254.255.2 via $GATEWAY_IP diff --git a/templates/worker_kubelet_flags.sh b/templates/worker_kubelet_flags.sh new file mode 100644 index 0000000..fd1a607 --- /dev/null +++ b/templates/worker_kubelet_flags.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +GREEN='\033[0;32m' # Color green +YELLOW='\033[0;33m' # Color yellow +NC='\033[0m' # No Color + +function wait_for_path() { + if [[ $2 == 'dir' ]]; then + while [ ! -d $1 ]; do + printf "$${YELLOW}Waiting for '$1' to be created...$${NC}\n" + sleep 10 + done + else + while [ ! -f $1 ]; do + printf "$${YELLOW}Waiting for '$1' to be created...$${NC}\n" + sleep 10 + done + fi + printf "$${GREEN}$1 FOUND!$${NC}\n" +} + +wait_for_path "/var/lib/kubelet/kubeadm-flags.env" + +sed -i '/KUBELET_KUBEADM_ARGS/ s/"$/ --cloud-provider=external"/' /var/lib/kubelet/kubeadm-flags.env +sudo systemctl restart kubelet + diff --git a/util/setup_gcp_project.sh b/util/setup_gcp_project.sh index 0dddbb6..4882f5f 100755 --- a/util/setup_gcp_project.sh +++ b/util/setup_gcp_project.sh @@ -1,6 +1,13 @@ #!/bin/bash +if ! command -v gcloud &> /dev/null +then + echo "The Google Cloud SDK is not installed." + echo "Pleaase visit https://cloud.google.com/sdk/docs/install for instructions on how to install." + exit 1 +fi + #start the interactive portion to capture user details echo "This script will help you perform the steps outlined at https://cloud.google.com/gke-on-prem/docs/how-to/service-accounts diff --git a/variables.tf b/variables.tf index ac6269e..d5f8718 100644 --- a/variables.tf +++ b/variables.tf @@ -5,9 +5,16 @@ variable "auth_token" { variable "project_id" { type = string + default = "null" description = "Equinix Metal Project ID" } +variable "organization_id" { + type = string + default = "null" + description = "Equinix Metal Organization ID" +} + variable "hostname" { type = string default = "anthos-baremetal" @@ -34,36 +41,81 @@ variable "worker_plan" { variable "ha_control_plane" { type = bool - description = "Do you want a highly available control plane" default = true + description = "Do you want a highly available control plane" } variable "worker_count" { type = number - description = "Number of baremetal worker nodes" default = 3 + description = "Number of baremetal worker nodes" } variable "operating_system" { type = string - description = "The Operating system of the node" default = "ubuntu_20_04" + description = "The Operating system of the node" } variable "billing_cycle" { type = string - description = "How the node will be billed (Not usually changed)" default = "hourly" + description = "How the node will be billed (Not usually changed)" } -variable "private_subnet" { +variable "cluster_name" { type = string - description = "Private IP Space to use for Layer2" - default = "172.29.254.0/24" + default = "equinix-metal-gke-cluster" + description = "The GKE cluster name" } -variable "cluster_name" { +variable "create_project" { + type = bool + default = true + description = "Create a Project if this is 'true'. Else use provided 'project_id'" +} + +variable "project_name" { type = string - description = "The GKE cluster name" - default = "equinix-metal-gke-cluster" + default = "baremetal-anthos" + description = "The name of the project if 'create_project' is 'true'." } + +# Advanced Variables below this line + +variable "bgp_asn" { + type = number + default = 65000 + description = "BGP ASN to peer with Equinix Metal" +} + +variable "ccm_version" { + type = string + default = "v2.0.0" + description = "The version of the Equinix Metal CCM" +} + +variable "kube_vip_version" { + type = string + default = "0.2.3" + description = "The version of Kube-VIP to use" +} + +variable "anthos_version" { + type = string + default = "1.6.0" + description = "The version of Google Anthos to install" +} + +variable "ccm_deploy_url" { + type = string + default = "https://gist.githubusercontent.com/thebsdbox/c86dd970549638105af8d96439175a59/raw/4abf90fb7929ded3f7a201818efbb6164b7081f0/ccm.yaml" + description = "The deploy url for the Equinix Metal CCM" +} + +variable "kube_vip_daemonset_url" { + type = string + default = "https://raw.githubusercontent.com/plunder-app/kube-vip/bb7d2da73eeb6c4712479b007ff931a12180e626/docs/manifests/kube-vip-em.yaml" + description = "The deploy url for the Kube-VIP Daemonset" +} +