From 8eb1b9359173438a6de03148ccf29bd729bec200 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Mon, 11 Jan 2021 12:50:08 -0800 Subject: [PATCH 01/30] Add updated examples for GKE and EKS In order to prevent confusion around progressive apply, provide some working examples of creating Kubernetes resources in the same apply where the cluster is created. --- _examples/eks/README.md | 56 ++++++++++ _examples/eks/kubernetes-config/main.tf | 101 ++++++++++++++++++ _examples/eks/kubernetes-config/variables.tf | 15 +++ _examples/eks/main.tf | 54 ++++++++++ _examples/eks/output.tf | 8 ++ _examples/eks/variables.tf | 21 ++++ _examples/eks/versions.tf | 4 + _examples/eks/vpc/main.tf | 68 ++++++++++++ _examples/eks/vpc/outputs.tf | 12 +++ _examples/gke/gke-cluster/main.tf | 37 +++++++ _examples/gke/gke-cluster/output.tf | 19 ++++ _examples/gke/gke-cluster/variables.tf | 11 ++ .../kubeconfig-template.yaml | 18 ++++ _examples/gke/kubernetes-config/main.tf | 74 +++++++++++++ _examples/gke/kubernetes-config/variables.tf | 16 +++ _examples/gke/main.tf | 33 ++++++ _examples/gke/outputs.tf | 7 ++ _examples/google-gke-cluster/main.tf | 2 +- kubernetes/test-infra/eks/main.tf | 2 +- .../test-infra/eks/node-config/config.tf | 5 +- 20 files changed, 558 insertions(+), 5 deletions(-) create mode 100644 _examples/eks/README.md create mode 100644 _examples/eks/kubernetes-config/main.tf create mode 100644 _examples/eks/kubernetes-config/variables.tf create mode 100644 _examples/eks/main.tf create mode 100644 _examples/eks/output.tf create mode 100644 _examples/eks/variables.tf create mode 100644 _examples/eks/versions.tf create mode 100644 _examples/eks/vpc/main.tf create mode 100644 _examples/eks/vpc/outputs.tf create mode 100644 _examples/gke/gke-cluster/main.tf create mode 100644 _examples/gke/gke-cluster/output.tf create mode 100644 _examples/gke/gke-cluster/variables.tf create mode 100644 _examples/gke/kubernetes-config/kubeconfig-template.yaml create mode 100644 _examples/gke/kubernetes-config/main.tf create mode 100644 _examples/gke/kubernetes-config/variables.tf create mode 100644 _examples/gke/main.tf create mode 100644 _examples/gke/outputs.tf diff --git a/_examples/eks/README.md b/_examples/eks/README.md new file mode 100644 index 0000000000..819d2c6677 --- /dev/null +++ b/_examples/eks/README.md @@ -0,0 +1,56 @@ +# Amazon EKS Clusters + +You will need the standard AWS environment variables to be set, e.g. + + - `AWS_ACCESS_KEY_ID` + - `AWS_SECRET_ACCESS_KEY` + +See [AWS Provider docs](https://www.terraform.io/docs/providers/aws/index.html#configuration-reference) for more details about these variables +and alternatives, like `AWS_PROFILE`. + +## Versions + +You can set the desired version of Kubernetes via the `kubernetes_version` TF variable. + +See https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html for currently available versions. + +You can set the desired version of Kubernetes via the `kubernetes_version` TF variable, like this: +``` +export TF_VAR_kubernetes_version="1.18" +``` +Alternatively you can pass it to the `apply` command line, like below. + +## Worker node count and instance type + +You can control the amount of worker nodes in the cluster as well as their machine type, using the following variables: + + - `TF_VAR_workers_count` + - `TF_VAR_workers_type` + +Export values for them or pass them to the apply command line. + +## Build the cluster + +``` +terraform init +terraform apply -var=kubernetes_version=1.19 +``` + +## Exporting K8S variables +To access the cluster you need to export the `KUBECONFIG` variable pointing to the `kubeconfig` file for the current cluster. +``` +export KUBECONFIG="$(terraform output kubeconfig_path)" +``` + +Now you can access the cluster via `kubectl` and you can run acceptance tests against it. + +To run acceptance tests, your the following command in the root of the repository. +``` +TESTARGS="-run '^TestAcc'" make testacc +``` + +To run only a specific set of tests, you can replace `^TestAcc` with any regular expression to filter tests by name. +For example, to run tests for Pod resources, you can do: +``` +TESTARGS="-run '^TestAccKubernetesPod_'" make testacc +``` diff --git a/_examples/eks/kubernetes-config/main.tf b/_examples/eks/kubernetes-config/main.tf new file mode 100644 index 0000000000..0db153a3cb --- /dev/null +++ b/_examples/eks/kubernetes-config/main.tf @@ -0,0 +1,101 @@ +data "aws_eks_cluster_auth" "cluster" { + name = var.cluster_name +} + +data "aws_eks_cluster" "cluster" { + name = var.cluster_name +} + +provider "kubernetes" { + host = var.cluster_endpoint + token = data.aws_eks_cluster_auth.cluster.token + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + exec { + api_version = "client.authentication.k8s.io/v1alpha1" + args = ["eks", "get-token", "--cluster-name", var.cluster_name] + command = "aws" + } +} + +locals { + mapped_role_format = < Date: Tue, 12 Jan 2021 12:53:57 -0800 Subject: [PATCH 02/30] add helm provider to examples --- _examples/eks/kubernetes-config/main.tf | 24 ++++++++++++++++++++++++ _examples/gke/kubernetes-config/main.tf | 20 ++++++++++++++++++++ _examples/gke/main.tf | 4 ++++ 3 files changed, 48 insertions(+) diff --git a/_examples/eks/kubernetes-config/main.tf b/_examples/eks/kubernetes-config/main.tf index 0db153a3cb..0fa2751a12 100644 --- a/_examples/eks/kubernetes-config/main.tf +++ b/_examples/eks/kubernetes-config/main.tf @@ -99,3 +99,27 @@ resource "kubernetes_deployment" "test" { } } } + +provider "helm" { + kubernetes { + host = var.cluster_endpoint + cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + exec { + api_version = "client.authentication.k8s.io/v1alpha1" + args = ["eks", "get-token", "--cluster-name", var.cluster_name] + command = "aws" + } + } +} + +resource helm_release nginx_ingress { + name = "nginx-ingress-controller" + + repository = "https://charts.bitnami.com/bitnami" + chart = "nginx-ingress-controller" + + set { + name = "service.type" + value = "ClusterIP" + } +} diff --git a/_examples/gke/kubernetes-config/main.tf b/_examples/gke/kubernetes-config/main.tf index 8babc1c305..19eab5a6a3 100644 --- a/_examples/gke/kubernetes-config/main.tf +++ b/_examples/gke/kubernetes-config/main.tf @@ -55,6 +55,26 @@ resource "kubernetes_deployment" "test" { } } +provider "helm" { + kubernetes { + host = var.cluster_endpoint + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(var.cluster_ca_cert) + } +} + +resource helm_release nginx_ingress { + name = "nginx-ingress-controller" + + repository = "https://charts.bitnami.com/bitnami" + chart = "nginx-ingress-controller" + + set { + name = "service.type" + value = "ClusterIP" + } +} + data "template_file" "kubeconfig" { template = file("${path.module}/kubeconfig-template.yaml") diff --git a/_examples/gke/main.tf b/_examples/gke/main.tf index 3af7ec0ccc..70faa2bafe 100644 --- a/_examples/gke/main.tf +++ b/_examples/gke/main.tf @@ -7,6 +7,10 @@ terraform { source = "hashicorp/google" version = "3.52" } + helm = { + source = "hashicorp/helm" + version = "2.0.1" + } } } From 38c421c0beb9e2a428a19b2cb91a19390971ea52 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 12 Jan 2021 13:46:31 -0800 Subject: [PATCH 03/30] Add README for GKE --- _examples/gke/README.md | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 _examples/gke/README.md diff --git a/_examples/gke/README.md b/_examples/gke/README.md new file mode 100644 index 0000000000..9f6aa9562b --- /dev/null +++ b/_examples/gke/README.md @@ -0,0 +1,50 @@ +# GKE (Google Container Engine) + +This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure a GKE cluster. The example builds the GKE cluster and applies the Kubernetes configurations in a single operation. + +You will need the following environment variables to be set: + + - `GOOGLE_CREDENTIALS` + - `GOOGLE_PROJECT` + - `GOOGLE_REGION` + +For example: +``` +$ env | grep GOOGLE +GOOGLE_REGION=us-west1 +GOOGLE_CREDENTIALS=/home/myuser/.config/gcloud/legacy_credentials/mygoogleuser/adc.json +GOOGLE_PROJECT=my-gcp-project +``` + +See [Google Provider docs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#full-reference) for more details about these variables. + +To install the GKE cluster using default values, run terraform init and apply from the directory containing this README. + +``` +terraform init +terraform apply +``` + +Optionally, the Kubernetes version can also be specified: + +``` +terraform apply -var=kubernetes_version=1.18 +``` + + +## Versions + +Valid versions for the GKE cluster can be found by using the gcloud tool. + +``` +gcloud container get-server-config --region $GOOGLE_REGION +``` + +## Kubeconfig for manual CLI access + +This example generates a kubeconfig file in the current working directory. However, the token in this config will expire after 1 hour. I can be refreshed by running `terraform apply` again. Alternatively, a longer-lived configuration can be generated using the gcloud tool. + +``` +gcloud container clusters get-credentials $(terraform output cluster_name) --region $GOOGLE_REGION +``` + From 182bbe622061f98ef7058a015e817087efd6f06f Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 12 Jan 2021 14:31:26 -0800 Subject: [PATCH 04/30] include instructions on generating kubeconfig --- _examples/eks/kubernetes-config/main.tf | 2 +- _examples/gke/README.md | 4 ++-- _examples/gke/gke-cluster/main.tf | 11 +++++++---- _examples/gke/gke-cluster/output.tf | 4 ++++ _examples/gke/gke-cluster/variables.tf | 4 ++++ _examples/gke/outputs.tf | 4 ++++ 6 files changed, 22 insertions(+), 7 deletions(-) diff --git a/_examples/eks/kubernetes-config/main.tf b/_examples/eks/kubernetes-config/main.tf index 0fa2751a12..0fd2714587 100644 --- a/_examples/eks/kubernetes-config/main.tf +++ b/_examples/eks/kubernetes-config/main.tf @@ -29,7 +29,7 @@ MAPPEDROLE } resource "kubernetes_config_map" "name" { - depends_on = [var.cluster_name] + depends_on = [var.cluster_name] metadata { name = "aws-auth" namespace = "kube-system" diff --git a/_examples/gke/README.md b/_examples/gke/README.md index 9f6aa9562b..4a6632736e 100644 --- a/_examples/gke/README.md +++ b/_examples/gke/README.md @@ -42,9 +42,9 @@ gcloud container get-server-config --region $GOOGLE_REGION ## Kubeconfig for manual CLI access -This example generates a kubeconfig file in the current working directory. However, the token in this config will expire after 1 hour. I can be refreshed by running `terraform apply` again. Alternatively, a longer-lived configuration can be generated using the gcloud tool. +This example generates a kubeconfig file in the current working directory. However, the token in this config will expire after 1 hour. I can be refreshed by running `terraform apply` again. Alternatively, a longer-lived configuration can be generated using the gcloud tool. Note: this command will overwrite the default kubeconfig at `$HOME/.kube/config`. ``` -gcloud container clusters get-credentials $(terraform output cluster_name) --region $GOOGLE_REGION +gcloud container clusters get-credentials $(terraform output cluster_name|jq -r) --zone $(terraform output google_zone |jq -r) ``` diff --git a/_examples/gke/gke-cluster/main.tf b/_examples/gke/gke-cluster/main.tf index be056bcb7c..87a8f88123 100644 --- a/_examples/gke/gke-cluster/main.tf +++ b/_examples/gke/gke-cluster/main.tf @@ -1,19 +1,22 @@ provider "google" { - // Provider settings to be provided via ENV variables + # Provider is configured using environment variables: GOOGLE_REGION, GOOGLE_PROJECT, GOOGLE_CREDENTIALS. + # This can be set statically, if preferred. See docs for details. + # https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#full-reference } +# This is used to set local variable google_zone. +# This can be replaced with a statically-configured zone, if preferred. data "google_compute_zones" "available" { } - data "google_container_engine_versions" "supported" { - location = data.google_compute_zones.available.names[0] + location = local.google_zone version_prefix = var.kubernetes_version } resource "google_container_cluster" "primary" { name = var.cluster_name - location = data.google_compute_zones.available.names[0] + location = local.google_zone initial_node_count = var.workers_count min_master_version = data.google_container_engine_versions.supported.latest_master_version # node version must match master version diff --git a/_examples/gke/gke-cluster/output.tf b/_examples/gke/gke-cluster/output.tf index bb6f191446..500403d0dc 100644 --- a/_examples/gke/gke-cluster/output.tf +++ b/_examples/gke/gke-cluster/output.tf @@ -17,3 +17,7 @@ output "cluster_ca_cert" { output "cluster_name" { value = google_container_cluster.primary.name } + +output "google_zone" { + value = local.google_zone +} diff --git a/_examples/gke/gke-cluster/variables.tf b/_examples/gke/gke-cluster/variables.tf index 81ddfba992..eb2f9f4488 100644 --- a/_examples/gke/gke-cluster/variables.tf +++ b/_examples/gke/gke-cluster/variables.tf @@ -9,3 +9,7 @@ variable "workers_count" { variable "cluster_name" { type = string } + +locals { + google_zone = data.google_compute_zones.available.names[0] +} diff --git a/_examples/gke/outputs.tf b/_examples/gke/outputs.tf index 18917bfee6..8eec4686f0 100644 --- a/_examples/gke/outputs.tf +++ b/_examples/gke/outputs.tf @@ -5,3 +5,7 @@ output "kubeconfig_path" { output "cluster_name" { value = local.cluster_name } + +output "google_zone" { + value = module.gke-cluster.google_zone +} From a8fabc69dc89b7bc33d0edd2a5a5dce6828a1a48 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 12 Jan 2021 15:25:27 -0800 Subject: [PATCH 05/30] working on EKS README --- _examples/eks/README.md | 55 ++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 31 deletions(-) diff --git a/_examples/eks/README.md b/_examples/eks/README.md index 819d2c6677..960936e3eb 100644 --- a/_examples/eks/README.md +++ b/_examples/eks/README.md @@ -1,24 +1,37 @@ -# Amazon EKS Clusters +# EKS (Amazon Elastic Kubernetes Service) -You will need the standard AWS environment variables to be set, e.g. +This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure an EKS cluster. The example builds the EKS cluster and applies the Kubernetes configurations in a single operation. + +You will need the following environment variables to be set: - `AWS_ACCESS_KEY_ID` - `AWS_SECRET_ACCESS_KEY` -See [AWS Provider docs](https://www.terraform.io/docs/providers/aws/index.html#configuration-reference) for more details about these variables -and alternatives, like `AWS_PROFILE`. +See [AWS Provider docs](https://www.terraform.io/docs/providers/aws/index.html#configuration-reference) for more details about these variables and alternatives, like `AWS_PROFILE`. + +To install the EKS cluster using default values, run terraform init and apply from the directory containing this README. + +``` +terraform init +terraform apply +``` -## Versions +Optionally, the Kubernetes version can also be specified: -You can set the desired version of Kubernetes via the `kubernetes_version` TF variable. +``` +terraform apply -var=kubernetes_version=1.18 +``` See https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html for currently available versions. -You can set the desired version of Kubernetes via the `kubernetes_version` TF variable, like this: +## Kubeconfig for manual CLI access + +This example generates a kubeconfig file in the current working directory. However, the token in this config expires in 15 minutes. I can be refreshed by running `terraform apply` again. Export the KUBECONFIG to manually access the cluster. + ``` -export TF_VAR_kubernetes_version="1.18" +export KUBECONFIG=$(terraform output kubeconfig_path) +kubectl get pods -n test ``` -Alternatively you can pass it to the `apply` command line, like below. ## Worker node count and instance type @@ -27,30 +40,10 @@ You can control the amount of worker nodes in the cluster as well as their machi - `TF_VAR_workers_count` - `TF_VAR_workers_type` -Export values for them or pass them to the apply command line. +Export values for them or pass them to the apply command line: -## Build the cluster - -``` -terraform init -terraform apply -var=kubernetes_version=1.19 -``` - -## Exporting K8S variables -To access the cluster you need to export the `KUBECONFIG` variable pointing to the `kubeconfig` file for the current cluster. ``` -export KUBECONFIG="$(terraform output kubeconfig_path)" +terraform apply -var=workers_count=4 -var=workers_type=m4.xlarge ``` -Now you can access the cluster via `kubectl` and you can run acceptance tests against it. -To run acceptance tests, your the following command in the root of the repository. -``` -TESTARGS="-run '^TestAcc'" make testacc -``` - -To run only a specific set of tests, you can replace `^TestAcc` with any regular expression to filter tests by name. -For example, to run tests for Pod resources, you can do: -``` -TESTARGS="-run '^TestAccKubernetesPod_'" make testacc -``` From 64f8ae0b0d022ac37f7fe4d01ea165843ee262c7 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 12 Jan 2021 15:35:06 -0800 Subject: [PATCH 06/30] update eks readme --- _examples/eks/README.md | 27 ++++++++++++--------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/_examples/eks/README.md b/_examples/eks/README.md index 960936e3eb..15834904b6 100644 --- a/_examples/eks/README.md +++ b/_examples/eks/README.md @@ -16,34 +16,31 @@ terraform init terraform apply ``` -Optionally, the Kubernetes version can also be specified: +## Kubeconfig for manual CLI access + +This example generates a kubeconfig file in the current working directory. However, the token in this config expires in 15 minutes. The token can be refreshed by running `terraform apply` again. Export the KUBECONFIG to manually access the cluster: ``` -terraform apply -var=kubernetes_version=1.18 +terraform apply +export KUBECONFIG=$(terraform output kubeconfig_path) +kubectl get pods -n test ``` -See https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html for currently available versions. +## Optional variables -## Kubeconfig for manual CLI access - -This example generates a kubeconfig file in the current working directory. However, the token in this config expires in 15 minutes. I can be refreshed by running `terraform apply` again. Export the KUBECONFIG to manually access the cluster. +The Kubernetes version can be specified at apply time: ``` -export KUBECONFIG=$(terraform output kubeconfig_path) -kubectl get pods -n test +terraform apply -var=kubernetes_version=1.18 ``` -## Worker node count and instance type +See https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html for currently available versions. -You can control the amount of worker nodes in the cluster as well as their machine type, using the following variables: - - `TF_VAR_workers_count` - - `TF_VAR_workers_type` +### Worker node count and instance type -Export values for them or pass them to the apply command line: +The number of worker nodes, and the instance type, can be specified at apply time: ``` terraform apply -var=workers_count=4 -var=workers_type=m4.xlarge ``` - - From fe4d74875f1feb9ed7cc11033b7ece0e31013c79 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 12 Jan 2021 15:39:03 -0800 Subject: [PATCH 07/30] update gke README --- _examples/gke/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_examples/gke/README.md b/_examples/gke/README.md index 4a6632736e..8bbd71dab7 100644 --- a/_examples/gke/README.md +++ b/_examples/gke/README.md @@ -42,7 +42,7 @@ gcloud container get-server-config --region $GOOGLE_REGION ## Kubeconfig for manual CLI access -This example generates a kubeconfig file in the current working directory. However, the token in this config will expire after 1 hour. I can be refreshed by running `terraform apply` again. Alternatively, a longer-lived configuration can be generated using the gcloud tool. Note: this command will overwrite the default kubeconfig at `$HOME/.kube/config`. +This example generates a kubeconfig file in the current working directory. However, the token in this config will expire after 1 hour. The token can be refreshed by running `terraform apply` again. Alternatively, a longer-lived configuration can be generated using the gcloud tool. Note: this command will overwrite the default kubeconfig at `$HOME/.kube/config`. ``` gcloud container clusters get-credentials $(terraform output cluster_name|jq -r) --zone $(terraform output google_zone |jq -r) From 365c3276018e68b6a8077aa591955faa05cd1a54 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 12 Jan 2021 16:09:48 -0800 Subject: [PATCH 08/30] clarify readme and remove unneeded vars from eks --- _examples/eks/README.md | 2 +- _examples/eks/kubernetes-config/main.tf | 27 ++++---------------- _examples/eks/kubernetes-config/variables.tf | 15 +++++++++-- _examples/eks/main.tf | 2 +- _examples/gke/README.md | 11 +++++++- 5 files changed, 30 insertions(+), 27 deletions(-) diff --git a/_examples/eks/README.md b/_examples/eks/README.md index 15834904b6..c487e5eb93 100644 --- a/_examples/eks/README.md +++ b/_examples/eks/README.md @@ -22,7 +22,7 @@ This example generates a kubeconfig file in the current working directory. Howev ``` terraform apply -export KUBECONFIG=$(terraform output kubeconfig_path) +export KUBECONFIG=$(terraform output kubeconfig_path|jq -r) kubectl get pods -n test ``` diff --git a/_examples/eks/kubernetes-config/main.tf b/_examples/eks/kubernetes-config/main.tf index 0fd2714587..d6e6a6dea0 100644 --- a/_examples/eks/kubernetes-config/main.tf +++ b/_examples/eks/kubernetes-config/main.tf @@ -1,15 +1,6 @@ -data "aws_eks_cluster_auth" "cluster" { - name = var.cluster_name -} - -data "aws_eks_cluster" "cluster" { - name = var.cluster_name -} - provider "kubernetes" { host = var.cluster_endpoint - token = data.aws_eks_cluster_auth.cluster.token - cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) + cluster_ca_certificate = base64decode(var.cluster_ca_cert) exec { api_version = "client.authentication.k8s.io/v1alpha1" args = ["eks", "get-token", "--cluster-name", var.cluster_name] @@ -17,17 +8,6 @@ provider "kubernetes" { } } -locals { - mapped_role_format = < Date: Tue, 12 Jan 2021 16:13:16 -0800 Subject: [PATCH 09/30] added link to EKS docs --- _examples/eks/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/_examples/eks/README.md b/_examples/eks/README.md index c487e5eb93..1a91283ea6 100644 --- a/_examples/eks/README.md +++ b/_examples/eks/README.md @@ -44,3 +44,7 @@ The number of worker nodes, and the instance type, can be specified at apply tim ``` terraform apply -var=workers_count=4 -var=workers_type=m4.xlarge ``` + +## Additional configuration of EKS + +To view all available configuration options for the EKS module used in this example, see [terraform-aws-modules/eks docs](https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest). From d689f89159c48e02eb48985f8cde56eb4d452655 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 12 Jan 2021 17:28:50 -0800 Subject: [PATCH 10/30] start adding AKS --- _examples/aks/README.md | 40 +++++++ _examples/aks/aks-cluster/main.tf | 64 ++++++++++ _examples/aks/aks-cluster/output.tf | 11 ++ _examples/aks/aks-cluster/variables.tf | 11 ++ _examples/aks/kubernetes-config/main.tf | 117 +++++++++++++++++++ _examples/aks/kubernetes-config/variables.tf | 1 + _examples/aks/main.tf | 35 ++++++ _examples/aks/outputs.tf | 7 ++ _examples/aks/variables.tf | 8 ++ _examples/gke/main.tf | 2 +- 10 files changed, 295 insertions(+), 1 deletion(-) create mode 100644 _examples/aks/README.md create mode 100644 _examples/aks/aks-cluster/main.tf create mode 100644 _examples/aks/aks-cluster/output.tf create mode 100644 _examples/aks/aks-cluster/variables.tf create mode 100644 _examples/aks/kubernetes-config/main.tf create mode 100644 _examples/aks/kubernetes-config/variables.tf create mode 100644 _examples/aks/main.tf create mode 100644 _examples/aks/outputs.tf create mode 100644 _examples/aks/variables.tf diff --git a/_examples/aks/README.md b/_examples/aks/README.md new file mode 100644 index 0000000000..54b95b32d4 --- /dev/null +++ b/_examples/aks/README.md @@ -0,0 +1,40 @@ +# AKS (Azure Kubernetes Service) + +This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure an AKS cluster. The example builds the AKS cluster and applies the Kubernetes configurations in a single operation. + +You will need the following environment variables to be set: + + - `ARM_SUBSCRIPTION_ID` + - `ARM_TENANT_ID` + - `ARM_CLIENT_ID` + - `ARM_CLIENT_SECRET` + +See [AWS Provider docs](https://www.terraform.io/docs/providers/aws/index.html#configuration-reference) for more details about these variables and alternatives, like `AWS_PROFILE`. + +To install the EKS cluster using default values, run terraform init and apply from the directory containing this README. + +``` +terraform init +terraform apply +``` + +## Kubeconfig for manual CLI access + +This example generates a kubeconfig file in the current working directory. However, the token in this config expires in 15 minutes. The token can be refreshed by running `terraform apply` again. Export the KUBECONFIG to manually access the cluster: + +``` +terraform apply +export KUBECONFIG=$(terraform output kubeconfig_path|jq -r) +kubectl get pods -n test +``` + +## Optional variables + +The Kubernetes version can be specified at apply time: + +``` +terraform apply -var=kubernetes_version=1.18 +``` + +See https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html for currently available versions. + diff --git a/_examples/aks/aks-cluster/main.tf b/_examples/aks/aks-cluster/main.tf new file mode 100644 index 0000000000..33bb383033 --- /dev/null +++ b/_examples/aks/aks-cluster/main.tf @@ -0,0 +1,64 @@ +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "test" { + name = local.cluster_name + location = var.location +} + +resource "azurerm_kubernetes_cluster" "test" { + name = local.cluster_name + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + dns_prefix = local.cluster_name + + default_node_pool { + name = "default" + node_count = 1 + vm_size = "Standard_DS2_v2" + } + + identity { + type = "SystemAssigned" + } + + addon_profile { + aci_connector_linux { + enabled = false + } + + azure_policy { + enabled = false + } + + http_application_routing { + enabled = false + } + + kube_dashboard { + enabled = true + } + + oms_agent { + enabled = false + } + } +} + +resource "local_file" "kubeconfig" { + content = azurerm_kubernetes_cluster.test.kube_config_raw + filename = "${path.module}/kubeconfig" +} + +resource "azurerm_managed_disk" "test" { + name = "testdisk" + location = azurerm_resource_group.test.location + resource_group_name = azurerm_resource_group.test.name + storage_account_type = "Standard_LRS" + create_option = "Empty" + disk_size_gb = "1" + tags = { + environment = azurerm_resource_group.test.name + } +} diff --git a/_examples/aks/aks-cluster/output.tf b/_examples/aks/aks-cluster/output.tf new file mode 100644 index 0000000000..5c90fbcd77 --- /dev/null +++ b/_examples/aks/aks-cluster/output.tf @@ -0,0 +1,11 @@ +output "cluster_ca_cert" { + value = azurerm_kubernetes_cluster.example.kube_config.0.client_certificate +} + +output "cluster_endpoint" { + value = azurerm_kubernetes_cluster.example.kube_config.0.endpoint +} + +output "data_disk_uri" { + value = azurerm_managed_disk.test.id +} diff --git a/_examples/aks/aks-cluster/variables.tf b/_examples/aks/aks-cluster/variables.tf new file mode 100644 index 0000000000..81ddfba992 --- /dev/null +++ b/_examples/aks/aks-cluster/variables.tf @@ -0,0 +1,11 @@ +variable "kubernetes_version" { + default = "1.18" +} + +variable "workers_count" { + default = "3" +} + +variable "cluster_name" { + type = string +} diff --git a/_examples/aks/kubernetes-config/main.tf b/_examples/aks/kubernetes-config/main.tf new file mode 100644 index 0000000000..c672975742 --- /dev/null +++ b/_examples/aks/kubernetes-config/main.tf @@ -0,0 +1,117 @@ +# This fetches a new token, which will expire in 1 hour. +data "azurerm_kubernetes_cluster" "main" { + name = var.cluster_name + resource_group_name = var.cluster_name +} + +provider "kubernetes" { + host = "${data.azurerm_kubernetes_cluster.main.kube_config.0.host}" + client_certificate = "${base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_certificate)}" + client_key = "${base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_key)}" + cluster_ca_certificate = "${base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate)}" +} + +resource "kubernetes_namespace" "test" { +depends_on = [var.cluster_name] + metadata { + name = "test" + } +} + +resource "kubernetes_persistent_volume" "test" { +depends_on = [var.cluster_name] + metadata { + name = "test" + } + spec { + capacity = { + storage = "1Gi" + } + access_modes = ["ReadWriteOnce"] + persistent_volume_source { + azure_disk { + caching_mode = "None" + data_disk_uri = var.disk_uri + disk_name = "managed" + kind = "Managed" + } + } + } +} + +resource "kubernetes_deployment" "test" { + metadata { + name = "test" + namespace= kubernetes_namespace.test.metadata.0.name + } + spec { + replicas = 2 + selector { + match_labels = { + TestLabelOne = "one" + } + } + template { + metadata { + labels = { + TestLabelOne = "one" + } + } + spec { + container { + image = "nginx:1.19.4" + name = "tf-acc-test" + + resources { + limits = { + memory = "512M" + cpu = "1" + } + requests = { + memory = "256M" + cpu = "50m" + } + } + } + } + } + } +} + +provider "helm" { + kubernetes { + host = var.cluster_endpoint + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(var.cluster_ca_cert) + } +} + +resource helm_release nginx_ingress { + name = "nginx-ingress-controller" + + repository = "https://charts.bitnami.com/bitnami" + chart = "nginx-ingress-controller" + + set { + name = "service.type" + value = "ClusterIP" + } +} + +data "template_file" "kubeconfig" { + template = file("${path.module}/kubeconfig-template.yaml") + + vars = { + cluster_name = var.cluster_name + endpoint = var.cluster_endpoint + cluster_ca = var.cluster_ca_cert + cluster_token = data.google_client_config.default.access_token + } +} + +resource "local_file" "kubeconfig" { + depends_on = [var.cluster_id] + content = data.template_file.kubeconfig.rendered + filename = "${path.root}/kubeconfig" +} + diff --git a/_examples/aks/kubernetes-config/variables.tf b/_examples/aks/kubernetes-config/variables.tf new file mode 100644 index 0000000000..58eb80dbc2 --- /dev/null +++ b/_examples/aks/kubernetes-config/variables.tf @@ -0,0 +1 @@ +azurerm_managed_disk.test.id diff --git a/_examples/aks/main.tf b/_examples/aks/main.tf new file mode 100644 index 0000000000..600ac63fdc --- /dev/null +++ b/_examples/aks/main.tf @@ -0,0 +1,35 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" +# version = "2.0" + } + google = { + source = "hashicorp/azure" + version = "2.42" + } + helm = { + source = "hashicorp/helm" + version = "2.0.1" + } + } +} + +resource "random_id" "cluster_name" { + byte_length = 5 +} + + +module "aks-cluster" { + source = "./aks-cluster" + cluster_name = local.cluster_name +} + +module "kubernetes-config" { + source = "./kubernetes-config" + cluster_name = module.aks-cluster.cluster_name + cluster_id = module.aks-cluster.cluster_id # creates dependency on cluster creation + cluster_endpoint = module.aks-cluster.cluster_endpoint + cluster_ca_cert = module.aks-cluster.cluster_ca_cert +} + diff --git a/_examples/aks/outputs.tf b/_examples/aks/outputs.tf new file mode 100644 index 0000000000..18917bfee6 --- /dev/null +++ b/_examples/aks/outputs.tf @@ -0,0 +1,7 @@ +output "kubeconfig_path" { + value = abspath("${path.root}/kubeconfig") +} + +output "cluster_name" { + value = local.cluster_name +} diff --git a/_examples/aks/variables.tf b/_examples/aks/variables.tf new file mode 100644 index 0000000000..e7e3d1990c --- /dev/null +++ b/_examples/aks/variables.tf @@ -0,0 +1,8 @@ +variable "location" { + type = string + default = "westus2" +} + +locals { + cluster_name = "tf-k8s-${random_id.cluster_name.hex}" +} diff --git a/_examples/gke/main.tf b/_examples/gke/main.tf index 70faa2bafe..397ebdcaea 100644 --- a/_examples/gke/main.tf +++ b/_examples/gke/main.tf @@ -19,7 +19,7 @@ resource "random_id" "cluster_name" { } locals { - cluster_name = "tf-acc-test-${random_id.cluster_name.hex}" + cluster_name = "tf-k8s-${random_id.cluster_name.hex}" } module "gke-cluster" { From ae739c6cda816d6229b14287735983b434860339 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Fri, 15 Jan 2021 13:12:11 -0800 Subject: [PATCH 11/30] validation passes --- _examples/aks/aks-cluster/main.tf | 28 ++----------- _examples/aks/aks-cluster/output.tf | 9 +++- _examples/aks/aks-cluster/variables.tf | 4 ++ _examples/aks/kubernetes-config/main.tf | 44 ++++++++------------ _examples/aks/kubernetes-config/variables.tf | 16 ++++++- _examples/aks/main.tf | 36 ++++++++-------- 6 files changed, 65 insertions(+), 72 deletions(-) diff --git a/_examples/aks/aks-cluster/main.tf b/_examples/aks/aks-cluster/main.tf index 33bb383033..658e1ad588 100644 --- a/_examples/aks/aks-cluster/main.tf +++ b/_examples/aks/aks-cluster/main.tf @@ -3,15 +3,15 @@ provider "azurerm" { } resource "azurerm_resource_group" "test" { - name = local.cluster_name + name = var.cluster_name location = var.location } resource "azurerm_kubernetes_cluster" "test" { - name = local.cluster_name + name = var.cluster_name location = azurerm_resource_group.test.location resource_group_name = azurerm_resource_group.test.name - dns_prefix = local.cluster_name + dns_prefix = var.cluster_name default_node_pool { name = "default" @@ -22,28 +22,6 @@ resource "azurerm_kubernetes_cluster" "test" { identity { type = "SystemAssigned" } - - addon_profile { - aci_connector_linux { - enabled = false - } - - azure_policy { - enabled = false - } - - http_application_routing { - enabled = false - } - - kube_dashboard { - enabled = true - } - - oms_agent { - enabled = false - } - } } resource "local_file" "kubeconfig" { diff --git a/_examples/aks/aks-cluster/output.tf b/_examples/aks/aks-cluster/output.tf index 5c90fbcd77..97df095129 100644 --- a/_examples/aks/aks-cluster/output.tf +++ b/_examples/aks/aks-cluster/output.tf @@ -1,11 +1,16 @@ output "cluster_ca_cert" { - value = azurerm_kubernetes_cluster.example.kube_config.0.client_certificate + value = azurerm_kubernetes_cluster.test.kube_config.0.client_certificate } output "cluster_endpoint" { - value = azurerm_kubernetes_cluster.example.kube_config.0.endpoint + value = azurerm_kubernetes_cluster.test.kube_config.0.host +} + +output "cluster_name" { + value = azurerm_kubernetes_cluster.test.id } output "data_disk_uri" { value = azurerm_managed_disk.test.id } + diff --git a/_examples/aks/aks-cluster/variables.tf b/_examples/aks/aks-cluster/variables.tf index 81ddfba992..0f37fe571c 100644 --- a/_examples/aks/aks-cluster/variables.tf +++ b/_examples/aks/aks-cluster/variables.tf @@ -9,3 +9,7 @@ variable "workers_count" { variable "cluster_name" { type = string } + +variable "location" { + type = string +} diff --git a/_examples/aks/kubernetes-config/main.tf b/_examples/aks/kubernetes-config/main.tf index c672975742..f38cd092db 100644 --- a/_examples/aks/kubernetes-config/main.tf +++ b/_examples/aks/kubernetes-config/main.tf @@ -1,14 +1,21 @@ -# This fetches a new token, which will expire in 1 hour. +provider "azurerm" { + features {} +} + +# The client certificate used for authenticating into the AKS cluster will eventually expire, +# (especially true if your clusters are created and destroyed periodically). +# This data source fetches new authentication certificates. +# Alternatively, use `terraform refresh` to fetch them manually. data "azurerm_kubernetes_cluster" "main" { name = var.cluster_name resource_group_name = var.cluster_name } provider "kubernetes" { - host = "${data.azurerm_kubernetes_cluster.main.kube_config.0.host}" - client_certificate = "${base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_certificate)}" - client_key = "${base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_key)}" - cluster_ca_certificate = "${base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate)}" + host = var.cluster_endpoint + cluster_ca_certificate = var.cluster_ca_cert + client_key = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_key) + client_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_certificate) } resource "kubernetes_namespace" "test" { @@ -31,7 +38,7 @@ depends_on = [var.cluster_name] persistent_volume_source { azure_disk { caching_mode = "None" - data_disk_uri = var.disk_uri + data_disk_uri = var.data_disk_uri disk_name = "managed" kind = "Managed" } @@ -80,9 +87,10 @@ resource "kubernetes_deployment" "test" { provider "helm" { kubernetes { - host = var.cluster_endpoint - token = data.google_client_config.default.access_token - cluster_ca_certificate = base64decode(var.cluster_ca_cert) + host = var.cluster_endpoint + client_certificate = var.cluster_ca_cert + client_key = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_key) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate) } } @@ -97,21 +105,3 @@ resource helm_release nginx_ingress { value = "ClusterIP" } } - -data "template_file" "kubeconfig" { - template = file("${path.module}/kubeconfig-template.yaml") - - vars = { - cluster_name = var.cluster_name - endpoint = var.cluster_endpoint - cluster_ca = var.cluster_ca_cert - cluster_token = data.google_client_config.default.access_token - } -} - -resource "local_file" "kubeconfig" { - depends_on = [var.cluster_id] - content = data.template_file.kubeconfig.rendered - filename = "${path.root}/kubeconfig" -} - diff --git a/_examples/aks/kubernetes-config/variables.tf b/_examples/aks/kubernetes-config/variables.tf index 58eb80dbc2..87858af261 100644 --- a/_examples/aks/kubernetes-config/variables.tf +++ b/_examples/aks/kubernetes-config/variables.tf @@ -1 +1,15 @@ -azurerm_managed_disk.test.id +variable "cluster_ca_cert" { + type = string +} + +variable "cluster_endpoint" { + type = string +} + +variable "cluster_name" { + type = string +} + +variable "data_disk_uri" { + type = string +} diff --git a/_examples/aks/main.tf b/_examples/aks/main.tf index 600ac63fdc..93f81a14b2 100644 --- a/_examples/aks/main.tf +++ b/_examples/aks/main.tf @@ -1,35 +1,37 @@ terraform { - required_providers { - kubernetes = { - source = "hashicorp/kubernetes" -# version = "2.0" - } - google = { - source = "hashicorp/azure" - version = "2.42" - } - helm = { - source = "hashicorp/helm" - version = "2.0.1" - } - } + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = "9.9.9" + } + azurerm = { + source = "hashicorp/azurerm" + version = "2.42" + } + helm = { + source = "hashicorp/helm" + version = "2.0.1" + } + } } resource "random_id" "cluster_name" { byte_length = 5 } - module "aks-cluster" { source = "./aks-cluster" cluster_name = local.cluster_name + location = var.location } +# By referencing the aks-cluster module as inputs to the kubernetes-config module, +# we establish a dependency between the two. This will create the AKS cluster before +# any Kubernetes resources are created. module "kubernetes-config" { source = "./kubernetes-config" cluster_name = module.aks-cluster.cluster_name - cluster_id = module.aks-cluster.cluster_id # creates dependency on cluster creation cluster_endpoint = module.aks-cluster.cluster_endpoint cluster_ca_cert = module.aks-cluster.cluster_ca_cert + data_disk_uri = module.aks-cluster.data_disk_uri } - From ac5be0272bafa5de82b2774b61cd197371ab081b Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Fri, 15 Jan 2021 15:36:41 -0800 Subject: [PATCH 12/30] apply works --- _examples/aks/aks-cluster/output.tf | 11 +---------- _examples/aks/kubernetes-config/main.tf | 14 ++++++++------ _examples/aks/kubernetes-config/variables.tf | 6 +----- _examples/aks/main.tf | 5 ++--- 4 files changed, 12 insertions(+), 24 deletions(-) diff --git a/_examples/aks/aks-cluster/output.tf b/_examples/aks/aks-cluster/output.tf index 97df095129..86fb3f53eb 100644 --- a/_examples/aks/aks-cluster/output.tf +++ b/_examples/aks/aks-cluster/output.tf @@ -1,16 +1,7 @@ -output "cluster_ca_cert" { - value = azurerm_kubernetes_cluster.test.kube_config.0.client_certificate -} - -output "cluster_endpoint" { - value = azurerm_kubernetes_cluster.test.kube_config.0.host -} - -output "cluster_name" { +output "cluster_id" { value = azurerm_kubernetes_cluster.test.id } output "data_disk_uri" { value = azurerm_managed_disk.test.id } - diff --git a/_examples/aks/kubernetes-config/main.tf b/_examples/aks/kubernetes-config/main.tf index f38cd092db..36496a865a 100644 --- a/_examples/aks/kubernetes-config/main.tf +++ b/_examples/aks/kubernetes-config/main.tf @@ -7,26 +7,27 @@ provider "azurerm" { # This data source fetches new authentication certificates. # Alternatively, use `terraform refresh` to fetch them manually. data "azurerm_kubernetes_cluster" "main" { +depends_on = [var.cluster_id] name = var.cluster_name resource_group_name = var.cluster_name } provider "kubernetes" { - host = var.cluster_endpoint - cluster_ca_certificate = var.cluster_ca_cert + host = data.azurerm_kubernetes_cluster.main.kube_config.0.host client_key = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_key) client_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_certificate) + cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate) } resource "kubernetes_namespace" "test" { -depends_on = [var.cluster_name] +depends_on = [var.cluster_id] metadata { name = "test" } } resource "kubernetes_persistent_volume" "test" { -depends_on = [var.cluster_name] +depends_on = [var.cluster_id] metadata { name = "test" } @@ -47,6 +48,7 @@ depends_on = [var.cluster_name] } resource "kubernetes_deployment" "test" { +depends_on = [var.cluster_id] metadata { name = "test" namespace= kubernetes_namespace.test.metadata.0.name @@ -87,9 +89,9 @@ resource "kubernetes_deployment" "test" { provider "helm" { kubernetes { - host = var.cluster_endpoint - client_certificate = var.cluster_ca_cert + host = data.azurerm_kubernetes_cluster.main.kube_config.0.host client_key = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_key) + client_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_certificate) cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate) } } diff --git a/_examples/aks/kubernetes-config/variables.tf b/_examples/aks/kubernetes-config/variables.tf index 87858af261..94abcbf591 100644 --- a/_examples/aks/kubernetes-config/variables.tf +++ b/_examples/aks/kubernetes-config/variables.tf @@ -1,8 +1,4 @@ -variable "cluster_ca_cert" { - type = string -} - -variable "cluster_endpoint" { +variable "cluster_id" { type = string } diff --git a/_examples/aks/main.tf b/_examples/aks/main.tf index 93f81a14b2..308364d163 100644 --- a/_examples/aks/main.tf +++ b/_examples/aks/main.tf @@ -30,8 +30,7 @@ module "aks-cluster" { # any Kubernetes resources are created. module "kubernetes-config" { source = "./kubernetes-config" - cluster_name = module.aks-cluster.cluster_name - cluster_endpoint = module.aks-cluster.cluster_endpoint - cluster_ca_cert = module.aks-cluster.cluster_ca_cert + cluster_id = module.aks-cluster.cluster_id + cluster_name = local.cluster_name data_disk_uri = module.aks-cluster.data_disk_uri } From c038a7b6865ff153afc9ebda31262f7fdbaf889b Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Fri, 15 Jan 2021 16:15:00 -0800 Subject: [PATCH 13/30] works in a single apply --- _examples/aks/kubernetes-config/main.tf | 1 + _examples/aks/main.tf | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/_examples/aks/kubernetes-config/main.tf b/_examples/aks/kubernetes-config/main.tf index 36496a865a..e02498491d 100644 --- a/_examples/aks/kubernetes-config/main.tf +++ b/_examples/aks/kubernetes-config/main.tf @@ -97,6 +97,7 @@ provider "helm" { } resource helm_release nginx_ingress { +depends_on = [var.cluster_id] name = "nginx-ingress-controller" repository = "https://charts.bitnami.com/bitnami" diff --git a/_examples/aks/main.tf b/_examples/aks/main.tf index 308364d163..ddb2dd0e4c 100644 --- a/_examples/aks/main.tf +++ b/_examples/aks/main.tf @@ -25,12 +25,9 @@ module "aks-cluster" { location = var.location } -# By referencing the aks-cluster module as inputs to the kubernetes-config module, -# we establish a dependency between the two. This will create the AKS cluster before -# any Kubernetes resources are created. module "kubernetes-config" { source = "./kubernetes-config" - cluster_id = module.aks-cluster.cluster_id + cluster_id = module.aks-cluster.cluster_id # creates dependency on cluster creation cluster_name = local.cluster_name data_disk_uri = module.aks-cluster.data_disk_uri } From 946df5f3d08182ec9042213c367f92b5097f568f Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Fri, 15 Jan 2021 16:35:09 -0800 Subject: [PATCH 14/30] fix kubeconfig path --- _examples/aks/aks-cluster/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_examples/aks/aks-cluster/main.tf b/_examples/aks/aks-cluster/main.tf index 658e1ad588..19e277ef34 100644 --- a/_examples/aks/aks-cluster/main.tf +++ b/_examples/aks/aks-cluster/main.tf @@ -26,7 +26,7 @@ resource "azurerm_kubernetes_cluster" "test" { resource "local_file" "kubeconfig" { content = azurerm_kubernetes_cluster.test.kube_config_raw - filename = "${path.module}/kubeconfig" + filename = "${path.root}/kubeconfig" } resource "azurerm_managed_disk" "test" { From 775a05ade1d9849198eb3b3fe49d598632d6e6c2 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Sun, 17 Jan 2021 11:32:03 -0800 Subject: [PATCH 15/30] figured out how to replace an AKS cluster --- _examples/aks/README.md | 29 +++++++---- _examples/aks/aks-cluster/main.tf | 7 +-- _examples/aks/aks-cluster/output.tf | 16 +++++- _examples/aks/kubernetes-config/main.tf | 55 +------------------- _examples/aks/kubernetes-config/variables.tf | 8 +-- _examples/aks/main.tf | 24 +++++++-- _examples/aks/variables.tf | 7 ++- 7 files changed, 68 insertions(+), 78 deletions(-) diff --git a/_examples/aks/README.md b/_examples/aks/README.md index 54b95b32d4..b5f3c011e3 100644 --- a/_examples/aks/README.md +++ b/_examples/aks/README.md @@ -1,6 +1,6 @@ # AKS (Azure Kubernetes Service) -This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure an AKS cluster. The example builds the AKS cluster and applies the Kubernetes configurations in a single operation. +This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure an AKS cluster. The example config in this directory builds the AKS cluster and applies the Kubernetes configurations in a single operation. This guide will also show you how to make changes to the underlying AKS cluster in such a way that Kuberntes/Helm resources are recreated after the underlying cluster is replaced. You will need the following environment variables to be set: @@ -11,7 +11,7 @@ You will need the following environment variables to be set: See [AWS Provider docs](https://www.terraform.io/docs/providers/aws/index.html#configuration-reference) for more details about these variables and alternatives, like `AWS_PROFILE`. -To install the EKS cluster using default values, run terraform init and apply from the directory containing this README. +To install the AKS cluster using default values, run terraform init and apply from the directory containing this README. ``` terraform init @@ -20,21 +20,32 @@ terraform apply ## Kubeconfig for manual CLI access -This example generates a kubeconfig file in the current working directory. However, the token in this config expires in 15 minutes. The token can be refreshed by running `terraform apply` again. Export the KUBECONFIG to manually access the cluster: +This example generates a kubeconfig file in the current working directory, which can be used for manual CLI access to the cluster. ``` -terraform apply export KUBECONFIG=$(terraform output kubeconfig_path|jq -r) kubectl get pods -n test ``` -## Optional variables - -The Kubernetes version can be specified at apply time: +However, in a real-world scenario, this config file would have to be replaced periodically as the AKS client certificates eventually expire (see the [Azure documentation](https://docs.microsoft.com/en-us/azure/aks/certificate-rotation) for the exact expiry dates). If the certificates are replaced, the AKS module will have to be targeted to pull in the new credentials before they can be passed into the Kubernetes or Helm providers. ``` -terraform apply -var=kubernetes_version=1.18 +terraform state rm module.kubernetes-config +terraform plan +terraform apply +export KUBECONFIG=$(terraform output kubeconfig_path|jq -r) +kubectl get pods -n test ``` -See https://docs.aws.amazon.com/eks/latest/userguide/platform-versions.html for currently available versions. +This approach prevents the Kubernetes and Helm provider from using cached, invalid credentials, which would cause provider configuration errors durring the plan and apply phases. (The resources that were previously deployed will not be affected by the `state rm`). + +## Replacing the AKS cluster, or its authentication credentials +When the cluster is initially created, the Kubernetes and Helm providers will not be initialized until authentication details are created for the cluster. However, for future operations that may involve replacing the underlying cluster (for example, changing VM sizes), the AKS cluster will have to be targeted without the Kubernetes/Helm providers, as shown below. This is done by removing the `module.kubernetes-config` from Terraform State prior to replacing cluster credentials, to avoid passing outdated credentials into the providers. + +This will create the new cluster and the Kubernetes resources in a single apply. If this is being applied to an existing cluster (such as in the case of credential rotation), the existing Kubernetes/Helm resources will continue running and simply undergo a credential refresh. + +``` +terraform state rm module.kubernetes-config +terraform apply +``` diff --git a/_examples/aks/aks-cluster/main.tf b/_examples/aks/aks-cluster/main.tf index 19e277ef34..7af26e9ac5 100644 --- a/_examples/aks/aks-cluster/main.tf +++ b/_examples/aks/aks-cluster/main.tf @@ -1,7 +1,3 @@ -provider "azurerm" { - features {} -} - resource "azurerm_resource_group" "test" { name = var.cluster_name location = var.location @@ -16,7 +12,8 @@ resource "azurerm_kubernetes_cluster" "test" { default_node_pool { name = "default" node_count = 1 - vm_size = "Standard_DS2_v2" + #vm_size = "Standard_DS2_v2" + vm_size = "Standard_A2_v2" } identity { diff --git a/_examples/aks/aks-cluster/output.tf b/_examples/aks/aks-cluster/output.tf index 86fb3f53eb..371982c26a 100644 --- a/_examples/aks/aks-cluster/output.tf +++ b/_examples/aks/aks-cluster/output.tf @@ -1,7 +1,19 @@ -output "cluster_id" { - value = azurerm_kubernetes_cluster.test.id +output "client_cert" { + value = azurerm_kubernetes_cluster.test.kube_config.0.client_certificate +} + +output "client_key" { + value = azurerm_kubernetes_cluster.test.kube_config.0.client_key +} + +output "ca_cert" { + value = azurerm_kubernetes_cluster.test.kube_config.0.cluster_ca_certificate } output "data_disk_uri" { value = azurerm_managed_disk.test.id } + +output "endpoint" { + value = azurerm_kubernetes_cluster.test.kube_config.0.host +} diff --git a/_examples/aks/kubernetes-config/main.tf b/_examples/aks/kubernetes-config/main.tf index e02498491d..c39c93320c 100644 --- a/_examples/aks/kubernetes-config/main.tf +++ b/_examples/aks/kubernetes-config/main.tf @@ -1,54 +1,10 @@ -provider "azurerm" { - features {} -} - -# The client certificate used for authenticating into the AKS cluster will eventually expire, -# (especially true if your clusters are created and destroyed periodically). -# This data source fetches new authentication certificates. -# Alternatively, use `terraform refresh` to fetch them manually. -data "azurerm_kubernetes_cluster" "main" { -depends_on = [var.cluster_id] - name = var.cluster_name - resource_group_name = var.cluster_name -} - -provider "kubernetes" { - host = data.azurerm_kubernetes_cluster.main.kube_config.0.host - client_key = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_key) - client_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_certificate) - cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate) -} - resource "kubernetes_namespace" "test" { -depends_on = [var.cluster_id] metadata { name = "test" } } -resource "kubernetes_persistent_volume" "test" { -depends_on = [var.cluster_id] - metadata { - name = "test" - } - spec { - capacity = { - storage = "1Gi" - } - access_modes = ["ReadWriteOnce"] - persistent_volume_source { - azure_disk { - caching_mode = "None" - data_disk_uri = var.data_disk_uri - disk_name = "managed" - kind = "Managed" - } - } - } -} - resource "kubernetes_deployment" "test" { -depends_on = [var.cluster_id] metadata { name = "test" namespace= kubernetes_namespace.test.metadata.0.name @@ -67,6 +23,7 @@ depends_on = [var.cluster_id] } } spec { + automount_service_account_token = false container { image = "nginx:1.19.4" name = "tf-acc-test" @@ -87,17 +44,7 @@ depends_on = [var.cluster_id] } } -provider "helm" { - kubernetes { - host = data.azurerm_kubernetes_cluster.main.kube_config.0.host - client_key = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_key) - client_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.client_certificate) - cluster_ca_certificate = base64decode(data.azurerm_kubernetes_cluster.main.kube_config.0.cluster_ca_certificate) - } -} - resource helm_release nginx_ingress { -depends_on = [var.cluster_id] name = "nginx-ingress-controller" repository = "https://charts.bitnami.com/bitnami" diff --git a/_examples/aks/kubernetes-config/variables.tf b/_examples/aks/kubernetes-config/variables.tf index 94abcbf591..17d3ab1c66 100644 --- a/_examples/aks/kubernetes-config/variables.tf +++ b/_examples/aks/kubernetes-config/variables.tf @@ -1,7 +1,7 @@ -variable "cluster_id" { - type = string -} - +#variable "client_cert" { +# type = string +#} +# variable "cluster_name" { type = string } diff --git a/_examples/aks/main.tf b/_examples/aks/main.tf index ddb2dd0e4c..0ef0748c33 100644 --- a/_examples/aks/main.tf +++ b/_examples/aks/main.tf @@ -15,19 +15,37 @@ terraform { } } -resource "random_id" "cluster_name" { - byte_length = 5 +provider "kubernetes" { + host = module.aks-cluster.endpoint + client_key = base64decode(module.aks-cluster.client_key) + client_certificate = base64decode(module.aks-cluster.client_cert) + cluster_ca_certificate = base64decode(module.aks-cluster.ca_cert) +} + +provider "helm" { + kubernetes { + host = module.aks-cluster.endpoint + client_key = base64decode(module.aks-cluster.client_key) + client_certificate = base64decode(module.aks-cluster.client_cert) + cluster_ca_certificate = base64decode(module.aks-cluster.ca_cert) + } +} + +provider "azurerm" { + features {} } module "aks-cluster" { + providers = { azurerm = azurerm } source = "./aks-cluster" cluster_name = local.cluster_name location = var.location } module "kubernetes-config" { + providers = { kubernetes = kubernetes, helm = helm } + depends_on = [module.aks-cluster] source = "./kubernetes-config" - cluster_id = module.aks-cluster.cluster_id # creates dependency on cluster creation cluster_name = local.cluster_name data_disk_uri = module.aks-cluster.data_disk_uri } diff --git a/_examples/aks/variables.tf b/_examples/aks/variables.tf index e7e3d1990c..9493cfd270 100644 --- a/_examples/aks/variables.tf +++ b/_examples/aks/variables.tf @@ -3,6 +3,11 @@ variable "location" { default = "westus2" } +resource "random_id" "cluster_name" { + byte_length = 5 +} + locals { - cluster_name = "tf-k8s-${random_id.cluster_name.hex}" + cluster_name = "tf-k8s-${random_id.cluster_name.hex}" + cluster_credentials_updated = timestamp() } From c3cfe9701da9ae67e60c07b744e9113efc934bde Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Sun, 17 Jan 2021 12:19:32 -0800 Subject: [PATCH 16/30] update readme --- _examples/aks/README.md | 10 ++++------ _examples/aks/aks-cluster/main.tf | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/_examples/aks/README.md b/_examples/aks/README.md index b5f3c011e3..e555ae8d20 100644 --- a/_examples/aks/README.md +++ b/_examples/aks/README.md @@ -27,23 +27,21 @@ export KUBECONFIG=$(terraform output kubeconfig_path|jq -r) kubectl get pods -n test ``` -However, in a real-world scenario, this config file would have to be replaced periodically as the AKS client certificates eventually expire (see the [Azure documentation](https://docs.microsoft.com/en-us/azure/aks/certificate-rotation) for the exact expiry dates). If the certificates are replaced, the AKS module will have to be targeted to pull in the new credentials before they can be passed into the Kubernetes or Helm providers. +However, in a real-world scenario, this config file would have to be replaced periodically as the AKS client certificates eventually expire (see the [Azure documentation](https://docs.microsoft.com/en-us/azure/aks/certificate-rotation) for the exact expiry dates). If the certificates (or other authentication attributes) are replaced, run `terraform apply` to pull in the new credentials. ``` -terraform state rm module.kubernetes-config -terraform plan terraform apply export KUBECONFIG=$(terraform output kubeconfig_path|jq -r) kubectl get pods -n test ``` -This approach prevents the Kubernetes and Helm provider from using cached, invalid credentials, which would cause provider configuration errors durring the plan and apply phases. (The resources that were previously deployed will not be affected by the `state rm`). +This approach prevents the Kubernetes and Helm providers from attempting to use cached, invalid credentials, which would cause provider configuration errors durring the plan and apply phases. -## Replacing the AKS cluster, or its authentication credentials +## Replacing the AKS cluster and re-creating the Kubernetes / Helm resources When the cluster is initially created, the Kubernetes and Helm providers will not be initialized until authentication details are created for the cluster. However, for future operations that may involve replacing the underlying cluster (for example, changing VM sizes), the AKS cluster will have to be targeted without the Kubernetes/Helm providers, as shown below. This is done by removing the `module.kubernetes-config` from Terraform State prior to replacing cluster credentials, to avoid passing outdated credentials into the providers. -This will create the new cluster and the Kubernetes resources in a single apply. If this is being applied to an existing cluster (such as in the case of credential rotation), the existing Kubernetes/Helm resources will continue running and simply undergo a credential refresh. +This will create the new cluster and the Kubernetes resources in a single apply. ``` terraform state rm module.kubernetes-config diff --git a/_examples/aks/aks-cluster/main.tf b/_examples/aks/aks-cluster/main.tf index 7af26e9ac5..ac2b60fc24 100644 --- a/_examples/aks/aks-cluster/main.tf +++ b/_examples/aks/aks-cluster/main.tf @@ -12,8 +12,8 @@ resource "azurerm_kubernetes_cluster" "test" { default_node_pool { name = "default" node_count = 1 - #vm_size = "Standard_DS2_v2" - vm_size = "Standard_A2_v2" + vm_size = "Standard_DS2_v2" + #vm_size = "Standard_A2_v2" } identity { From 9dac85a5e54527b38371410e711b322eacd46d4c Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Sun, 17 Jan 2021 12:33:54 -0800 Subject: [PATCH 17/30] update readmes --- _examples/aks/README.md | 7 ++++++- _examples/aks/aks-cluster/main.tf | 1 - _examples/eks/README.md | 7 +++++++ _examples/gke/README.md | 7 +++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/_examples/aks/README.md b/_examples/aks/README.md index e555ae8d20..f5e0fe645f 100644 --- a/_examples/aks/README.md +++ b/_examples/aks/README.md @@ -9,7 +9,12 @@ You will need the following environment variables to be set: - `ARM_CLIENT_ID` - `ARM_CLIENT_SECRET` -See [AWS Provider docs](https://www.terraform.io/docs/providers/aws/index.html#configuration-reference) for more details about these variables and alternatives, like `AWS_PROFILE`. +Ensure that `KUBE_CONFIG_FILE` and `KUBE_CONFIG_FILES` environment variables are NOT set, as they will interfere with the cluster build. + +``` +unset KUBE_CONFIG_FILE +unset KUBE_CONFIG_FILES +``` To install the AKS cluster using default values, run terraform init and apply from the directory containing this README. diff --git a/_examples/aks/aks-cluster/main.tf b/_examples/aks/aks-cluster/main.tf index ac2b60fc24..9ced747e23 100644 --- a/_examples/aks/aks-cluster/main.tf +++ b/_examples/aks/aks-cluster/main.tf @@ -13,7 +13,6 @@ resource "azurerm_kubernetes_cluster" "test" { name = "default" node_count = 1 vm_size = "Standard_DS2_v2" - #vm_size = "Standard_A2_v2" } identity { diff --git a/_examples/eks/README.md b/_examples/eks/README.md index 1a91283ea6..2a0c430507 100644 --- a/_examples/eks/README.md +++ b/_examples/eks/README.md @@ -9,6 +9,13 @@ You will need the following environment variables to be set: See [AWS Provider docs](https://www.terraform.io/docs/providers/aws/index.html#configuration-reference) for more details about these variables and alternatives, like `AWS_PROFILE`. +Ensure that `KUBE_CONFIG_FILE` and `KUBE_CONFIG_FILES` environment variables are NOT set, as they will interfere with the cluster build. + +``` +unset KUBE_CONFIG_FILE +unset KUBE_CONFIG_FILES +``` + To install the EKS cluster using default values, run terraform init and apply from the directory containing this README. ``` diff --git a/_examples/gke/README.md b/_examples/gke/README.md index 294cd019bd..ba30015823 100644 --- a/_examples/gke/README.md +++ b/_examples/gke/README.md @@ -18,6 +18,13 @@ GOOGLE_PROJECT=my-gcp-project See [Google Provider docs](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#full-reference) for more details about these variables. +Ensure that `KUBE_CONFIG_FILE` and `KUBE_CONFIG_FILES` environment variables are NOT set, as they will interfere with the cluster build. + +``` +unset KUBE_CONFIG_FILE +unset KUBE_CONFIG_FILES +``` + To install the GKE cluster using default values, run terraform init and apply from the directory containing this README. ``` From 076030f08ce62b111a60db31979ca0e79132755f Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Sun, 17 Jan 2021 13:21:46 -0800 Subject: [PATCH 18/30] update readme --- _examples/eks/README.md | 11 +++++++++++ _examples/gke/README.md | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/_examples/eks/README.md b/_examples/eks/README.md index 2a0c430507..054a05e255 100644 --- a/_examples/eks/README.md +++ b/_examples/eks/README.md @@ -55,3 +55,14 @@ terraform apply -var=workers_count=4 -var=workers_type=m4.xlarge ## Additional configuration of EKS To view all available configuration options for the EKS module used in this example, see [terraform-aws-modules/eks docs](https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest). + +## Replacing the EKS cluster and re-creating the Kubernetes / Helm resources + +When the cluster is initially created, the Kubernetes and Helm providers will not be initialized until authentication details are created for the cluster. However, for future operations that may involve replacing the underlying cluster (for example, changing VM sizes), the EKS cluster will have to be targeted without the Kubernetes/Helm providers, as shown below. This is done by removing the `module.kubernetes-config` from Terraform State prior to replacing cluster credentials, to avoid passing outdated credentials into the providers. + +This will create the new cluster and the Kubernetes resources in a single apply. + +``` +terraform state rm module.kubernetes-config +terraform apply +``` diff --git a/_examples/gke/README.md b/_examples/gke/README.md index ba30015823..810c919f7a 100644 --- a/_examples/gke/README.md +++ b/_examples/gke/README.md @@ -64,3 +64,13 @@ gcloud container clusters get-credentials $(terraform output cluster_name|jq -r) kubectl get pods -n test ``` +## Replacing the GKE cluster and re-creating the Kubernetes / Helm resources + +When the cluster is initially created, the Kubernetes and Helm providers will not be initialized until authentication details are created for the cluster. However, for future operations that may involve replacing the underlying cluster (for example, changing VM sizes), the GKE cluster will have to be targeted without the Kubernetes/Helm providers, as shown below. This is done by removing the `module.kubernetes-config` from Terraform State prior to replacing cluster credentials, to avoid passing outdated credentials into the providers. + +This will create the new cluster and the Kubernetes resources in a single apply. + +``` +terraform state rm module.kubernetes-config +terraform apply +``` From 69751d6d0ac2e735a70ceea2d9dad9a0e653be6c Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Mon, 18 Jan 2021 12:47:14 -0800 Subject: [PATCH 19/30] update readme and add helm provider --- _examples/eks/README.md | 2 +- _examples/eks/main.tf | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/_examples/eks/README.md b/_examples/eks/README.md index 054a05e255..b20eaa0a85 100644 --- a/_examples/eks/README.md +++ b/_examples/eks/README.md @@ -58,7 +58,7 @@ To view all available configuration options for the EKS module used in this exam ## Replacing the EKS cluster and re-creating the Kubernetes / Helm resources -When the cluster is initially created, the Kubernetes and Helm providers will not be initialized until authentication details are created for the cluster. However, for future operations that may involve replacing the underlying cluster (for example, changing VM sizes), the EKS cluster will have to be targeted without the Kubernetes/Helm providers, as shown below. This is done by removing the `module.kubernetes-config` from Terraform State prior to replacing cluster credentials, to avoid passing outdated credentials into the providers. +When the cluster is initially created, the Kubernetes and Helm providers will not be initialized until authentication details are created for the cluster. However, for future operations that may involve replacing the underlying cluster (for example, changing the network where the EKS cluster resides), the EKS cluster will have to be targeted without the Kubernetes/Helm providers, as shown below. This is done by removing the `module.kubernetes-config` from Terraform State prior to replacing cluster credentials, to avoid passing outdated credentials into the providers. This will create the new cluster and the Kubernetes resources in a single apply. diff --git a/_examples/eks/main.tf b/_examples/eks/main.tf index 87096349b0..5436811c24 100644 --- a/_examples/eks/main.tf +++ b/_examples/eks/main.tf @@ -3,6 +3,10 @@ terraform { kubernetes = { source = "hashicorp/kubernetes" } + helm = { + source = "hashicorp/helm" + version = "2.0.1" + } aws = { source = "hashicorp/aws" version = "3.22.0" From b8997b74c361b0ec2db6c5e9043db84ca7254e4c Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 19 Jan 2021 14:36:37 -0800 Subject: [PATCH 20/30] update readme to remove jq --- _examples/aks/README.md | 4 ++-- _examples/aks/aks-cluster/output.tf | 4 ---- _examples/aks/kubernetes-config/variables.tf | 8 -------- _examples/aks/main.tf | 1 - _examples/eks/README.md | 2 +- _examples/gke/README.md | 4 ++-- 6 files changed, 5 insertions(+), 18 deletions(-) diff --git a/_examples/aks/README.md b/_examples/aks/README.md index f5e0fe645f..3e56888723 100644 --- a/_examples/aks/README.md +++ b/_examples/aks/README.md @@ -28,7 +28,7 @@ terraform apply This example generates a kubeconfig file in the current working directory, which can be used for manual CLI access to the cluster. ``` -export KUBECONFIG=$(terraform output kubeconfig_path|jq -r) +export KUBECONFIG=$(terraform output -raw kubeconfig_path) kubectl get pods -n test ``` @@ -36,7 +36,7 @@ However, in a real-world scenario, this config file would have to be replaced pe ``` terraform apply -export KUBECONFIG=$(terraform output kubeconfig_path|jq -r) +export KUBECONFIG=$(terraform output -raw kubeconfig_path) kubectl get pods -n test ``` diff --git a/_examples/aks/aks-cluster/output.tf b/_examples/aks/aks-cluster/output.tf index 371982c26a..9bb8518983 100644 --- a/_examples/aks/aks-cluster/output.tf +++ b/_examples/aks/aks-cluster/output.tf @@ -10,10 +10,6 @@ output "ca_cert" { value = azurerm_kubernetes_cluster.test.kube_config.0.cluster_ca_certificate } -output "data_disk_uri" { - value = azurerm_managed_disk.test.id -} - output "endpoint" { value = azurerm_kubernetes_cluster.test.kube_config.0.host } diff --git a/_examples/aks/kubernetes-config/variables.tf b/_examples/aks/kubernetes-config/variables.tf index 17d3ab1c66..abbf86f798 100644 --- a/_examples/aks/kubernetes-config/variables.tf +++ b/_examples/aks/kubernetes-config/variables.tf @@ -1,11 +1,3 @@ -#variable "client_cert" { -# type = string -#} -# variable "cluster_name" { type = string } - -variable "data_disk_uri" { - type = string -} diff --git a/_examples/aks/main.tf b/_examples/aks/main.tf index 0ef0748c33..0be4044bcc 100644 --- a/_examples/aks/main.tf +++ b/_examples/aks/main.tf @@ -47,5 +47,4 @@ module "kubernetes-config" { depends_on = [module.aks-cluster] source = "./kubernetes-config" cluster_name = local.cluster_name - data_disk_uri = module.aks-cluster.data_disk_uri } diff --git a/_examples/eks/README.md b/_examples/eks/README.md index b20eaa0a85..c65d1059f0 100644 --- a/_examples/eks/README.md +++ b/_examples/eks/README.md @@ -29,7 +29,7 @@ This example generates a kubeconfig file in the current working directory. Howev ``` terraform apply -export KUBECONFIG=$(terraform output kubeconfig_path|jq -r) +export KUBECONFIG=$(terraform output -raw kubeconfig_path) kubectl get pods -n test ``` diff --git a/_examples/gke/README.md b/_examples/gke/README.md index 810c919f7a..5f1faa79d8 100644 --- a/_examples/gke/README.md +++ b/_examples/gke/README.md @@ -53,14 +53,14 @@ This example generates a kubeconfig file in the current working directory. Howev ``` terraform apply -export KUBECONFIG=$(terraform output kubeconfig_path|jq -r) +export KUBECONFIG=$(terraform output -raw kubeconfig_path) kubectl get pods -n test ``` Alternatively, a longer-lived configuration can be generated using the gcloud tool. Note: this command will overwrite the default kubeconfig at `$HOME/.kube/config`. ``` -gcloud container clusters get-credentials $(terraform output cluster_name|jq -r) --zone $(terraform output google_zone |jq -r) +gcloud container clusters get-credentials $(terraform output -raw cluster_name) --zone $(terraform output -raw google_zone) kubectl get pods -n test ``` From 0cf0cf3c31130598220a250d6dbd2a8fd0e846d4 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 19 Jan 2021 15:14:57 -0800 Subject: [PATCH 21/30] remove version until 2.0 is released --- _examples/aks/main.tf | 1 - 1 file changed, 1 deletion(-) diff --git a/_examples/aks/main.tf b/_examples/aks/main.tf index 0be4044bcc..653d6aabbd 100644 --- a/_examples/aks/main.tf +++ b/_examples/aks/main.tf @@ -2,7 +2,6 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" - version = "9.9.9" } azurerm = { source = "hashicorp/azurerm" From 5276ddb0fd1479a9e4492d6e57151ccc781a09c5 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 19 Jan 2021 15:38:02 -0800 Subject: [PATCH 22/30] added more details to AKS readme --- _examples/aks/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/_examples/aks/README.md b/_examples/aks/README.md index 3e56888723..5340fcc574 100644 --- a/_examples/aks/README.md +++ b/_examples/aks/README.md @@ -32,12 +32,18 @@ export KUBECONFIG=$(terraform output -raw kubeconfig_path) kubectl get pods -n test ``` -However, in a real-world scenario, this config file would have to be replaced periodically as the AKS client certificates eventually expire (see the [Azure documentation](https://docs.microsoft.com/en-us/azure/aks/certificate-rotation) for the exact expiry dates). If the certificates (or other authentication attributes) are replaced, run `terraform apply` to pull in the new credentials. +However, in a real-world scenario, this config file would have to be replaced periodically as the AKS client certificates eventually expire (see the [Azure documentation](https://docs.microsoft.com/en-us/azure/aks/certificate-rotation) for the exact expiry dates). If the certificates (or other authentication attributes) are replaced, run a targeted `terraform apply` to save the new credentials into state. ``` +terraform plan -target=module.aks-cluster +terraform apply -target=module.aks-cluster +``` + +Once the targeted apply is finished, the Kubernetes and Helm providers will be available for use again. Run `terraform apply` again (without targeting) to apply any updates to Kubernetes resources. + +``` +terraform plan terraform apply -export KUBECONFIG=$(terraform output -raw kubeconfig_path) -kubectl get pods -n test ``` This approach prevents the Kubernetes and Helm providers from attempting to use cached, invalid credentials, which would cause provider configuration errors durring the plan and apply phases. From 0e027b1d23fb236fb270737480120c97837e4baa Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 19 Jan 2021 16:17:16 -0800 Subject: [PATCH 23/30] remove unneeded disk --- _examples/aks/aks-cluster/main.tf | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/_examples/aks/aks-cluster/main.tf b/_examples/aks/aks-cluster/main.tf index 9ced747e23..1a0648910b 100644 --- a/_examples/aks/aks-cluster/main.tf +++ b/_examples/aks/aks-cluster/main.tf @@ -25,14 +25,3 @@ resource "local_file" "kubeconfig" { filename = "${path.root}/kubeconfig" } -resource "azurerm_managed_disk" "test" { - name = "testdisk" - location = azurerm_resource_group.test.location - resource_group_name = azurerm_resource_group.test.name - storage_account_type = "Standard_LRS" - create_option = "Empty" - disk_size_gb = "1" - tags = { - environment = azurerm_resource_group.test.name - } -} From 1b017b39f8f49e1cfff76b6a8754bd3e3c556b08 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Tue, 19 Jan 2021 16:29:32 -0800 Subject: [PATCH 24/30] minor fixes --- _examples/aks/kubernetes-config/main.tf | 7 +++---- _examples/aks/variables.tf | 1 - _examples/eks/README.md | 2 +- _examples/eks/kubernetes-config/main.tf | 7 +++---- _examples/gke/README.md | 2 +- _examples/gke/kubernetes-config/main.tf | 6 +++--- 6 files changed, 11 insertions(+), 14 deletions(-) diff --git a/_examples/aks/kubernetes-config/main.tf b/_examples/aks/kubernetes-config/main.tf index c39c93320c..3eeabc804d 100644 --- a/_examples/aks/kubernetes-config/main.tf +++ b/_examples/aks/kubernetes-config/main.tf @@ -13,20 +13,19 @@ resource "kubernetes_deployment" "test" { replicas = 2 selector { match_labels = { - TestLabelOne = "one" + app = "test" } } template { metadata { labels = { - TestLabelOne = "one" + app = "test" } } spec { - automount_service_account_token = false container { image = "nginx:1.19.4" - name = "tf-acc-test" + name = "nginx" resources { limits = { diff --git a/_examples/aks/variables.tf b/_examples/aks/variables.tf index 9493cfd270..74ca370a9c 100644 --- a/_examples/aks/variables.tf +++ b/_examples/aks/variables.tf @@ -9,5 +9,4 @@ resource "random_id" "cluster_name" { locals { cluster_name = "tf-k8s-${random_id.cluster_name.hex}" - cluster_credentials_updated = timestamp() } diff --git a/_examples/eks/README.md b/_examples/eks/README.md index c65d1059f0..37255e745a 100644 --- a/_examples/eks/README.md +++ b/_examples/eks/README.md @@ -1,6 +1,6 @@ # EKS (Amazon Elastic Kubernetes Service) -This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure an EKS cluster. The example builds the EKS cluster and applies the Kubernetes configurations in a single operation. +This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure an EKS cluster. The example config builds the EKS cluster and applies the Kubernetes configurations in a single operation. You will need the following environment variables to be set: diff --git a/_examples/eks/kubernetes-config/main.tf b/_examples/eks/kubernetes-config/main.tf index d6e6a6dea0..2c963afcbd 100644 --- a/_examples/eks/kubernetes-config/main.tf +++ b/_examples/eks/kubernetes-config/main.tf @@ -34,7 +34,6 @@ resource "null_resource" "generate-kubeconfig" { } } - resource "kubernetes_namespace" "test" { depends_on = [var.cluster_name] metadata { @@ -52,19 +51,19 @@ resource "kubernetes_deployment" "test" { replicas = 2 selector { match_labels = { - TestLabelOne = "one" + app = "test" } } template { metadata { labels = { - TestLabelOne = "one" + app = "test" } } spec { container { image = "nginx:1.19.4" - name = "tf-acc-test" + name = "nginx" resources { limits = { diff --git a/_examples/gke/README.md b/_examples/gke/README.md index 5f1faa79d8..0f66444c53 100644 --- a/_examples/gke/README.md +++ b/_examples/gke/README.md @@ -1,6 +1,6 @@ # GKE (Google Container Engine) -This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure a GKE cluster. The example builds the GKE cluster and applies the Kubernetes configurations in a single operation. +This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure a GKE cluster. The example config builds the GKE cluster and applies the Kubernetes configurations in a single operation. You will need the following environment variables to be set: diff --git a/_examples/gke/kubernetes-config/main.tf b/_examples/gke/kubernetes-config/main.tf index 19eab5a6a3..ac3ef4effc 100644 --- a/_examples/gke/kubernetes-config/main.tf +++ b/_examples/gke/kubernetes-config/main.tf @@ -25,19 +25,19 @@ resource "kubernetes_deployment" "test" { replicas = 2 selector { match_labels = { - TestLabelOne = "one" + app = "test" } } template { metadata { labels = { - TestLabelOne = "one" + app = "test" } } spec { container { image = "nginx:1.19.4" - name = "tf-acc-test" + name = "nginx" resources { limits = { From 40008c97cfce74b4bf005025afc8b86b0d1ea505 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Wed, 20 Jan 2021 13:57:34 -0800 Subject: [PATCH 25/30] update or remove old examples --- _examples/google-gke-cluster/README.md | 130 -------------- _examples/google-gke-cluster/main.tf | 72 -------- _examples/http-nginx/README.md | 68 ------- .../http-nginx/replication-controller.tf | 45 ----- _examples/http-nginx/service.tf | 22 --- _examples/http-nginx/variables.tf | 4 - _examples/ingress/README.md | 9 +- _examples/ingress/main.tf | 170 +++--------------- _examples/job/main.tf | 34 ++-- 9 files changed, 58 insertions(+), 496 deletions(-) delete mode 100644 _examples/google-gke-cluster/README.md delete mode 100644 _examples/google-gke-cluster/main.tf delete mode 100644 _examples/http-nginx/README.md delete mode 100644 _examples/http-nginx/replication-controller.tf delete mode 100644 _examples/http-nginx/service.tf delete mode 100644 _examples/http-nginx/variables.tf diff --git a/_examples/google-gke-cluster/README.md b/_examples/google-gke-cluster/README.md deleted file mode 100644 index 5a8668f730..0000000000 --- a/_examples/google-gke-cluster/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# Google GKE (Google Container Engine) cluster - -In case you don't have a K8S cluster yet the easiest way -to create one from scratch is to use GKE (Google Container Service). - -You can read more about GKE at https://cloud.google.com/container-engine/ - -## Prerequisites - -*This example uses syntax elements specific to Terraform version 0.12+. -It will not work out-of-the-box with Terraform 0.11.x and lower.* - -Configure the Google Cloud provider by supplying environment variables -and/or standard config files. -Check out [related docs](https://www.terraform.io/docs/providers/google/index.html#configuration-reference) -on how to do so, specifically look at arguments `credentials` and `project`. - -## Creating cluster - -First we make sure the Google provider is downloaded and available - -```sh -terraform init -``` - -then we carry on by creating the real infrastructure which -requires region, cluster username and password. - -```sh -terraform apply \ - -var 'region=us-west1' \ - -var 'username=MySecretUsername' \ - -var 'password=MySecretPassword' -``` - -You may also specify the K8S version (see [available versions](https://cloud.google.com/container-engine/release-notes)) -and name of the cluster. - -```sh -terraform apply \ - -var 'kubernetes_version=1.16.8' \ - -var 'cluster_name=terraform-example-cluster' \ - -var 'region=us-west1' \ - -var 'username=MySecretUsername' \ - -var 'password=MySecretPassword' -``` - -Afterwards you should see output similar to this one in your console - -``` -... - -Outputs: - -additional_zones = [ - us-west1-b -] -cluster_name = terraform-example-cluster -endpoint = 102.186.121.2 -node_version = 1.16.8 -primary_zone = us-west1-a -``` - -## Credentials - -It is generally a good practice not to hard-code credentials -in your source code and use environment variables and/or standard config instead. -Check out [the relevant docs](https://www.terraform.io/docs/providers/kubernetes/index.html#argument-reference) -for all supported provider arguments, most of which are related to authentication. - -Once you have a cluster up and running on GKE the easiest way to supply -credentials to the provider is via the following set of steps: - -```sh -gcloud container clusters get-credentials \ - $(terraform output cluster_name) \ - --zone=$(terraform output primary_zone) -``` - -Afterwards you should have a set of valid credentials stored -in the config at default location where the Kubernetes provider -can find them. The current context will also be automatically -pointed to those new credentials (in case you have any other -credentials there in an existing config). - -You can verify this by running - -```sh -kubectl cluster-info -``` - -which should provide output similar to the one below - -``` -Kubernetes master is running at https://102.186.121.2 -GLBCDefaultBackend is running at https://102.186.121.2/api/v1/namespaces/kube-system/services/default-http-backend/proxy -Heapster is running at https://102.186.121.2/api/v1/namespaces/kube-system/services/heapster/proxy -KubeDNS is running at https://102.186.121.2/api/v1/namespaces/kube-system/services/kube-dns/proxy -kubernetes-dashboard is running at https://102.186.121.2/api/v1/namespaces/kube-system/services/kubernetes-dashboard/proxy - -To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'. -``` - -You can also visit the Kubernetes dashboard by opening a proxy - -```sh -kubectl proxy -``` - -which will print out the IP & port on which the proxy is listening - -``` -Starting to serve on 127.0.0.1:8001 -``` - -then you can either copy & paste that into your browser -or just do this in a separate console session (while keeping the proxy running) - -``` -open http://127.0.0.1:8001/ui/ -``` - -## Destroying cluster - -```sh -terraform destroy \ - -var 'region=us-west1' \ - -var 'username=' \ - -var 'password=' -``` diff --git a/_examples/google-gke-cluster/main.tf b/_examples/google-gke-cluster/main.tf deleted file mode 100644 index 17971bb089..0000000000 --- a/_examples/google-gke-cluster/main.tf +++ /dev/null @@ -1,72 +0,0 @@ -variable "region" { -} - -provider "google" { - region = var.region - // Provider settings to be provided via ENV variables -} - -data "google_compute_zones" "available" { -} - -variable "cluster_name" { - default = "terraform-example-cluster" -} - -variable "kubernetes_version" { - default = "1.18" -} - -variable "username" { -} - -variable "password" { -} - -resource "google_container_cluster" "primary" { - name = var.cluster_name - location = data.google_compute_zones.available.names[0] - initial_node_count = 3 - - min_master_version = var.kubernetes_version - node_version = var.kubernetes_version - - node_locations = [ - data.google_compute_zones.available.names[1], - ] - - master_auth { - username = var.username - password = var.password - } - - node_config { - oauth_scopes = [ - "https://www.googleapis.com/auth/compute", - "https://www.googleapis.com/auth/devstorage.read_only", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring", - ] - } -} - -output "cluster_name" { - value = google_container_cluster.primary.name -} - -output "primary_zone" { - value = google_container_cluster.primary.zone -} - -output "additional_zones" { - value = google_container_cluster.primary.additional_zones -} - -output "endpoint" { - value = google_container_cluster.primary.endpoint -} - -output "node_version" { - value = google_container_cluster.primary.node_version -} - diff --git a/_examples/http-nginx/README.md b/_examples/http-nginx/README.md deleted file mode 100644 index bb9fb4c8f2..0000000000 --- a/_examples/http-nginx/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# Example: NGINX (HTTP) - -This example is heavily inspired by https://github.com/kubernetes/examples/tree/master/staging/https-nginx - -It shows how to spin up a basic HTTP server on Kubernetes using [nginx](https://www.nginx.com) -which is exposed to the internet through a load balancer (provisioned automatically by K8S). - -## Used resources - - - `kubernetes_replication_controller` - - `kubernetes_service` - -## Prerequisites - -*This example uses syntax elements specific to Terraform version 0.12+. -It will not work out-of-the-box with Terraform 0.11.x and lower.* - -This example expects you to already have a running K8S cluster -and credentials set up in a config or environment variables. - -See [related docs](../google-gke-cluster/README.md) if you don't have any of those. - -## How to - -### Create - -First we make sure the Kubernetes provider is downloaded and available - -```sh -terraform init -``` - -then we carry on by creating the resources - -```sh -terraform apply -``` - -you may optionally specify the version of nginx like this - -```sh -terraform apply -var 'nginx_version=1.7.8' -``` - -After the `apply` operation has finished you should see output -in your console similar to the one below - -``` -... - -Outputs: - -lb_ip = 35.197.9.247 -``` - -This is the IP address of your public load balancer -which exposes the web server. Open that IP in your -browser to see the nginx welcome page. - -```sh -open "http://$(terraform output lb_ip)" -``` - -### Destroy - -``` -terraform destroy -``` diff --git a/_examples/http-nginx/replication-controller.tf b/_examples/http-nginx/replication-controller.tf deleted file mode 100644 index c1352c5fe0..0000000000 --- a/_examples/http-nginx/replication-controller.tf +++ /dev/null @@ -1,45 +0,0 @@ -resource "kubernetes_replication_controller" "example" { - metadata { - name = "terraform-nginx-example" - labels = { - App = "TerraformNginxExample" - } - } - - spec { - selector = { - App = "TerraformNginxExample" - } - template { - container { - image = "nginx:${var.nginx_version}" - name = "example" - - port { - container_port = 80 - } - - liveness_probe { - http_get { - path = "/index.html" - port = 80 - } - initial_delay_seconds = 30 - timeout_seconds = 1 - } - - resources { - limits = { - cpu = "0.5" - memory = "512Mi" - } - requests = { - cpu = "250m" - memory = "50Mi" - } - } - } - } - } -} - diff --git a/_examples/http-nginx/service.tf b/_examples/http-nginx/service.tf deleted file mode 100644 index 800338af2d..0000000000 --- a/_examples/http-nginx/service.tf +++ /dev/null @@ -1,22 +0,0 @@ -resource "kubernetes_service" "example" { - metadata { - name = "terraform-nginx-example" - } - spec { - selector = { - App = kubernetes_replication_controller.example.metadata[0].labels.App - } - session_affinity = "ClientIP" - port { - port = 80 - target_port = 80 - } - - type = "LoadBalancer" - } -} - -output "lb_ip" { - value = kubernetes_service.example.load_balancer_ingress[0].ip -} - diff --git a/_examples/http-nginx/variables.tf b/_examples/http-nginx/variables.tf deleted file mode 100644 index 46a34d3273..0000000000 --- a/_examples/http-nginx/variables.tf +++ /dev/null @@ -1,4 +0,0 @@ -variable "nginx_version" { - default = "1.7.8" -} - diff --git a/_examples/ingress/README.md b/_examples/ingress/README.md index dc08868c0b..87e394e8d8 100644 --- a/_examples/ingress/README.md +++ b/_examples/ingress/README.md @@ -1,6 +1,9 @@ -# Example: Ingress +# Example: Ingress with AWS ELB ## Prerequisites -*This example uses syntax elements specific to Terraform version 0.12+. -It will not work out-of-the-box with Terraform 0.11.x and lower.* +* This example uses syntax elements specific to Terraform version 0.12+. +It will not work out-of-the-box with Terraform 0.11.x and lower. + +* This example uses annotations specific to the [AWS Load Balancer Controller](https://docs.aws.amazon.com/eks/latest/userguide/aws-load-balancer-controller.html). + diff --git a/_examples/ingress/main.tf b/_examples/ingress/main.tf index be0671e6bd..eb594f661c 100644 --- a/_examples/ingress/main.tf +++ b/_examples/ingress/main.tf @@ -1,166 +1,54 @@ -provider "kubernetes" { -} - -resource "kubernetes_ingress" "example" { - metadata { - name = "example" - - annotations = { - "ingress.kubernetes.io/rewrite-target" = "/" +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" } } +} - spec { - backend { - service_name = "echoserver" - service_port = 8080 - } - - rule { - host = "myminikube.info" - - http { - path { - path = "/" - - backend { - service_name = "echoserver" - service_port = 8080 - } - } - } - } - - rule { - host = "cheeses.all" - - http { - path { - path = "/stilton" - - backend { - service_name = "stilton-cheese" - service_port = 80 - } - } - - path { - path = "/cheddar" - - backend { - service_name = "cheddar" - service_port = 80 - } - } - } - } +resource "kubernetes_namespace" "test" { + metadata { + name = "test" + namespace = "test" } } -resource "kubernetes_service" "echoserver" { +resource "kubernetes_service" "test" { metadata { - name = "echoserver" + name = "test" + namespace = kubernetes_namespace.test.metadata.0.name } - spec { - selector = { - app = "echoserver" - } - port { - port = 8080 - target_port = 8080 + port = 80 + target_port = 80 + protocol = "TCP" } - type = "NodePort" } } -resource "kubernetes_deployment" "echoserver" { +resource "kubernetes_ingress" "test" { metadata { - name = "echoserver" - } - - spec { - selector { - match_labels = { - app = "echoserver" - } - } - - template { - metadata { - labels = { - app = "echoserver" - } - } - - spec { - container { - name = "echoserver" - image = "gcr.io/google_containers/echoserver:1.4" - - port { - container_port = 8080 - } - } - } + name = "test" + namespace = kubernetes_namespace.test.metadata.0.name + annotations = { + "kubernetes.io/ingress.class" = "alb" + "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "alb.ingress.kubernetes.io/target-type" = "ip" } } -} - -resource "kubernetes_deployment" "cheddar" { - metadata { - name = "cheddar-cheese" - } - spec { - selector { - match_labels = { - app = "cheddar" - } - } - - template { - metadata { - labels = { - app = "cheddar" - } - } - - spec { - container { - name = "cheddar" - image = "errm/cheese:cheddar" - - port { - container_port = 80 + rule { + http { + path { + path = "/*" + backend { + service_name = kubernetes_service.test.metadata.0.name + service_port = 80 } } } } } } - -resource "kubernetes_service" "cheddar" { - metadata { - name = "cheddar" - } - - spec { - selector = { - app = "cheddar" - } - - port { - port = 80 - target_port = 80 - } - - type = "NodePort" - } -} - -output "ingress_ip" { - value = formatlist("%s ", kubernetes_ingress.example.load_balancer_ingress.*.ip) -} diff --git a/_examples/job/main.tf b/_examples/job/main.tf index 81ee8b4a86..95ef5efe13 100644 --- a/_examples/job/main.tf +++ b/_examples/job/main.tf @@ -1,27 +1,39 @@ +terraform { + required_providers { + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.0" + } + } +} + +variable "kube_config_file" { + default = "~/.kube/config" +} + provider "kubernetes" { + config_file = var.kube_config_file } -resource "kubernetes_job" "test-pr" { +resource "kubernetes_job" "test" { metadata { - name = "job-with-wait" - namespace = "default" + name = "test" } spec { - completions = 1 + active_deadline_seconds = 120 + backoff_limit = 10 + completions = 10 + parallelism = 2 template { metadata {} spec { container { - name = "sleep" - image = "busybox:latest" + name = "hello" + image = "busybox" command = ["sleep", "30"] } - restart_policy = "Never" } } } wait_for_completion = true - timeouts { - create = "40s" - } -} \ No newline at end of file +} From da99ae37fab6c0f707551bf4a92be300a8818d44 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Thu, 21 Jan 2021 10:40:47 -0800 Subject: [PATCH 26/30] automate fixing formatting in examples --- GNUmakefile | 12 ++++ _examples/aks/main.tf | 18 +++--- _examples/aks/variables.tf | 4 +- _examples/certificate-signing-request/main.tf | 10 ++-- .../certificate-signing-request/variables.tf | 8 +-- _examples/eks/main.tf | 12 ++-- _examples/gke/main.tf | 16 +++--- _examples/google-gke-nfs-filestore/main.tf | 26 ++++----- _examples/ingress/main.tf | 16 +++--- _examples/job/main.tf | 10 ++-- scripts/fmt-examples.sh | 55 +++++++++++++++++++ 11 files changed, 127 insertions(+), 60 deletions(-) create mode 100755 scripts/fmt-examples.sh diff --git a/GNUmakefile b/GNUmakefile index 660de6637c..4f2e07573d 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -33,6 +33,18 @@ depscheck: @git diff --exit-code -- vendor || \ (echo; echo "Unexpected difference in vendor/ directory. Run 'go mod vendor' command or revert any go.mod/go.sum/vendor changes and commit."; exit 1) +examples-lint: tools + @echo "==> Checking _examples dir formatting..." + @./scripts/fmt-examples.sh || (echo; \ + echo "Terraform formatting errors found in _examples dir."; \ + echo "To see the full differences, run: ./scripts/fmt-examples.sh diff"; \ + echo "To automatically fix the formatting, run 'make examples-lint-fix' and commit the changes."; \ + exit 1) + +examples-lint-fix: tools + @echo "==> Fixing terraform formatting of _examples dir..." + @./scripts/fmt-examples.sh fix + fmt: gofmt -w $(GOFMT_FILES) diff --git a/_examples/aks/main.tf b/_examples/aks/main.tf index 653d6aabbd..0a42d037f4 100644 --- a/_examples/aks/main.tf +++ b/_examples/aks/main.tf @@ -1,7 +1,7 @@ terraform { required_providers { kubernetes = { - source = "hashicorp/kubernetes" + source = "hashicorp/kubernetes" } azurerm = { source = "hashicorp/azurerm" @@ -35,15 +35,15 @@ provider "azurerm" { } module "aks-cluster" { - providers = { azurerm = azurerm } - source = "./aks-cluster" - cluster_name = local.cluster_name - location = var.location + providers = { azurerm = azurerm } + source = "./aks-cluster" + cluster_name = local.cluster_name + location = var.location } module "kubernetes-config" { - providers = { kubernetes = kubernetes, helm = helm } - depends_on = [module.aks-cluster] - source = "./kubernetes-config" - cluster_name = local.cluster_name + providers = { kubernetes = kubernetes, helm = helm } + depends_on = [module.aks-cluster] + source = "./kubernetes-config" + cluster_name = local.cluster_name } diff --git a/_examples/aks/variables.tf b/_examples/aks/variables.tf index 74ca370a9c..e8d47c0d96 100644 --- a/_examples/aks/variables.tf +++ b/_examples/aks/variables.tf @@ -1,5 +1,5 @@ variable "location" { - type = string + type = string default = "westus2" } @@ -8,5 +8,5 @@ resource "random_id" "cluster_name" { } locals { - cluster_name = "tf-k8s-${random_id.cluster_name.hex}" + cluster_name = "tf-k8s-${random_id.cluster_name.hex}" } diff --git a/_examples/certificate-signing-request/main.tf b/_examples/certificate-signing-request/main.tf index 828b41ebae..323e6b36f2 100644 --- a/_examples/certificate-signing-request/main.tf +++ b/_examples/certificate-signing-request/main.tf @@ -1,6 +1,6 @@ resource "tls_private_key" "example" { algorithm = "ECDSA" - rsa_bits = "4096" + rsa_bits = "4096" } resource "tls_cert_request" "example" { @@ -19,7 +19,7 @@ resource "kubernetes_certificate_signing_request" "example" { } spec { request = tls_cert_request.example.cert_request_pem - usages = ["client auth", "server auth"] + usages = ["client auth", "server auth"] } auto_approve = true } @@ -41,12 +41,12 @@ resource "kubernetes_pod" "main" { } spec { container { - name = "default" - image = "alpine:latest" + name = "default" + image = "alpine:latest" command = ["cat", "/etc/test/tls.crt"] volume_mount { mount_path = "/etc/test" - name = "secretvol" + name = "secretvol" } } volume { diff --git a/_examples/certificate-signing-request/variables.tf b/_examples/certificate-signing-request/variables.tf index 203e53a28e..e81b4aaff6 100644 --- a/_examples/certificate-signing-request/variables.tf +++ b/_examples/certificate-signing-request/variables.tf @@ -1,7 +1,7 @@ -variable example_user { - default = "admin" +variable "example_user" { + default = "admin" } -variable example_org { - default = "example cluster" +variable "example_org" { + default = "example cluster" } diff --git a/_examples/eks/main.tf b/_examples/eks/main.tf index 5436811c24..6bd7cb769b 100644 --- a/_examples/eks/main.tf +++ b/_examples/eks/main.tf @@ -1,7 +1,7 @@ terraform { required_providers { kubernetes = { - source = "hashicorp/kubernetes" + source = "hashicorp/kubernetes" } helm = { source = "hashicorp/helm" @@ -50,9 +50,9 @@ module "cluster" { } module "kubernetes-config" { - source = "./kubernetes-config" - k8s_node_role_arn = list(module.cluster.worker_iam_role_arn) - cluster_ca_cert = module.cluster.cluster_certificate_authority_data - cluster_name = module.cluster.cluster_id # creates dependency on cluster creation - cluster_endpoint = module.cluster.cluster_endpoint + source = "./kubernetes-config" + k8s_node_role_arn = list(module.cluster.worker_iam_role_arn) + cluster_ca_cert = module.cluster.cluster_certificate_authority_data + cluster_name = module.cluster.cluster_id # creates dependency on cluster creation + cluster_endpoint = module.cluster.cluster_endpoint } diff --git a/_examples/gke/main.tf b/_examples/gke/main.tf index 397ebdcaea..67a69f1e60 100644 --- a/_examples/gke/main.tf +++ b/_examples/gke/main.tf @@ -1,7 +1,7 @@ terraform { required_providers { kubernetes = { - source = "hashicorp/kubernetes" + source = "hashicorp/kubernetes" } google = { source = "hashicorp/google" @@ -23,15 +23,15 @@ locals { } module "gke-cluster" { - source = "./gke-cluster" - cluster_name = local.cluster_name + source = "./gke-cluster" + cluster_name = local.cluster_name } module "kubernetes-config" { - source = "./kubernetes-config" - cluster_name = module.gke-cluster.cluster_name - cluster_id = module.gke-cluster.cluster_id # creates dependency on cluster creation - cluster_endpoint = module.gke-cluster.cluster_endpoint - cluster_ca_cert = module.gke-cluster.cluster_ca_cert + source = "./kubernetes-config" + cluster_name = module.gke-cluster.cluster_name + cluster_id = module.gke-cluster.cluster_id # creates dependency on cluster creation + cluster_endpoint = module.gke-cluster.cluster_endpoint + cluster_ca_cert = module.gke-cluster.cluster_ca_cert } diff --git a/_examples/google-gke-nfs-filestore/main.tf b/_examples/google-gke-nfs-filestore/main.tf index 3a5927eca6..91f8e6b25f 100644 --- a/_examples/google-gke-nfs-filestore/main.tf +++ b/_examples/google-gke-nfs-filestore/main.tf @@ -26,8 +26,8 @@ variable "workers_count" { } data "google_container_engine_versions" "supported" { - location = data.google_compute_zones.available.names[0] - version_prefix = var.kubernetes_version + location = data.google_compute_zones.available.names[0] + version_prefix = var.kubernetes_version } # If the result is empty '[]', the GKE default_cluster_version will be used. @@ -37,7 +37,7 @@ output "available_master_versions_matching_user_input" { # Shared network for GKE cluster and Filestore to use. resource "google_compute_network" "vpc" { - name = "shared" + name = "shared" auto_create_subnetworks = true } @@ -49,7 +49,7 @@ resource "google_container_cluster" "primary" { min_master_version = data.google_container_engine_versions.supported.latest_master_version # node version must match master version # https://www.terraform.io/docs/providers/google/r/container_cluster.html#node_version - node_version = data.google_container_engine_versions.supported.latest_master_version + node_version = data.google_container_engine_versions.supported.latest_master_version node_locations = [ data.google_compute_zones.available.names[1], @@ -109,7 +109,7 @@ resource "local_file" "kubeconfig" { } provider "kubernetes" { - version = "1.11.2" + version = "1.11.2" load_config_file = "false" host = google_container_cluster.primary.endpoint @@ -131,7 +131,7 @@ resource "kubernetes_storage_class" "nfs" { metadata { name = "filestore" } - reclaim_policy = "Retain" + reclaim_policy = "Retain" storage_provisioner = "nfs" } @@ -144,11 +144,11 @@ resource "kubernetes_persistent_volume" "example" { storage = "1T" } storage_class_name = kubernetes_storage_class.nfs.metadata[0].name - access_modes = ["ReadWriteMany"] + access_modes = ["ReadWriteMany"] persistent_volume_source { nfs { server = google_filestore_instance.test.networks[0].ip_addresses[0] - path = "/${google_filestore_instance.test.file_shares[0].name}" + path = "/${google_filestore_instance.test.file_shares[0].name}" } } } @@ -156,13 +156,13 @@ resource "kubernetes_persistent_volume" "example" { resource "kubernetes_persistent_volume_claim" "example" { metadata { - name = "mariadb-data" + name = "mariadb-data" namespace = "test" } spec { - access_modes = ["ReadWriteMany"] + access_modes = ["ReadWriteMany"] storage_class_name = kubernetes_storage_class.nfs.metadata[0].name - volume_name = kubernetes_persistent_volume.example.metadata[0].name + volume_name = kubernetes_persistent_volume.example.metadata[0].name resources { requests = { storage = "1T" @@ -173,7 +173,7 @@ resource "kubernetes_persistent_volume_claim" "example" { resource "kubernetes_deployment" "mariadb" { metadata { - name = "mariadb-example" + name = "mariadb-example" namespace = "test" labels = { mylabel = "MyExampleApp" @@ -202,7 +202,7 @@ resource "kubernetes_deployment" "mariadb" { name = "example" env { - name = "MYSQL_RANDOM_ROOT_PASSWORD" + name = "MYSQL_RANDOM_ROOT_PASSWORD" value = true } diff --git a/_examples/ingress/main.tf b/_examples/ingress/main.tf index eb594f661c..d7d9628b80 100644 --- a/_examples/ingress/main.tf +++ b/_examples/ingress/main.tf @@ -1,28 +1,28 @@ terraform { required_providers { kubernetes = { - source = "hashicorp/kubernetes" + source = "hashicorp/kubernetes" } } } resource "kubernetes_namespace" "test" { metadata { - name = "test" + name = "test" namespace = "test" } } resource "kubernetes_service" "test" { metadata { - name = "test" + name = "test" namespace = kubernetes_namespace.test.metadata.0.name } spec { port { - port = 80 + port = 80 target_port = 80 - protocol = "TCP" + protocol = "TCP" } type = "NodePort" } @@ -30,11 +30,11 @@ resource "kubernetes_service" "test" { resource "kubernetes_ingress" "test" { metadata { - name = "test" + name = "test" namespace = kubernetes_namespace.test.metadata.0.name annotations = { - "kubernetes.io/ingress.class" = "alb" - "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "kubernetes.io/ingress.class" = "alb" + "alb.ingress.kubernetes.io/scheme" = "internet-facing" "alb.ingress.kubernetes.io/target-type" = "ip" } } diff --git a/_examples/job/main.tf b/_examples/job/main.tf index 95ef5efe13..5697b4e3cb 100644 --- a/_examples/job/main.tf +++ b/_examples/job/main.tf @@ -21,15 +21,15 @@ resource "kubernetes_job" "test" { } spec { active_deadline_seconds = 120 - backoff_limit = 10 - completions = 10 - parallelism = 2 + backoff_limit = 10 + completions = 10 + parallelism = 2 template { metadata {} spec { container { - name = "hello" - image = "busybox" + name = "hello" + image = "busybox" command = ["sleep", "30"] } } diff --git a/scripts/fmt-examples.sh b/scripts/fmt-examples.sh new file mode 100755 index 0000000000..76e72690f4 --- /dev/null +++ b/scripts/fmt-examples.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Check examples dir for formatting errors. +set -o errexit +set -o errtrace +set -o nounset +set -o pipefail + +#-------------- +# Functions +#-------------- + +function formatting_check() { + cd _examples + for dir in $(ls); do + cd ${dir} + terraform fmt --check --list=false || (echo "Formatting errors found in dir: _examples/${dir}"; exit 1) + cd - + done +} + +function formatting_diff() { + cd _examples + for dir in $(ls); do + cd ${dir} + terraform fmt --check -diff || (echo "Formatting errors found in dir: _examples/${dir}"; exit 1) + cd - + done +} + +function formatting_fix() { + cd _examples + for dir in $(ls); do + cd ${dir} + terraform fmt + cd - + done +} + +#-------------- +# Main +#-------------- + +input="${1:-}" + +case ${input} in +diff) + formatting_diff + ;; +fix) + formatting_fix + ;; +*) + formatting_check + ;; +esac From 403e63bc7a758eff290cdc1d16032c19de0e55ec Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Thu, 21 Jan 2021 10:52:56 -0800 Subject: [PATCH 27/30] add some version constraints --- _examples/aks/main.tf | 3 ++- _examples/eks/main.tf | 3 ++- _examples/gke/main.tf | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/_examples/aks/main.tf b/_examples/aks/main.tf index 0a42d037f4..91e08dfaac 100644 --- a/_examples/aks/main.tf +++ b/_examples/aks/main.tf @@ -2,6 +2,7 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" + version = ">= 2.0.0" } azurerm = { source = "hashicorp/azurerm" @@ -9,7 +10,7 @@ terraform { } helm = { source = "hashicorp/helm" - version = "2.0.1" + version = ">= 2.0.1" } } } diff --git a/_examples/eks/main.tf b/_examples/eks/main.tf index 6bd7cb769b..927b7f1457 100644 --- a/_examples/eks/main.tf +++ b/_examples/eks/main.tf @@ -2,10 +2,11 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" + version = ">= 2.0.0" } helm = { source = "hashicorp/helm" - version = "2.0.1" + version = ">= 2.0.1" } aws = { source = "hashicorp/aws" diff --git a/_examples/gke/main.tf b/_examples/gke/main.tf index 67a69f1e60..34647aa911 100644 --- a/_examples/gke/main.tf +++ b/_examples/gke/main.tf @@ -2,6 +2,7 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" + version = ">= 2.0.0" } google = { source = "hashicorp/google" @@ -9,7 +10,7 @@ terraform { } helm = { source = "hashicorp/helm" - version = "2.0.1" + version = ">= 2.0.1" } } } From ee738a5e8e6f233796f48d366f00042ea1ec8c2e Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Thu, 21 Jan 2021 12:11:49 -0800 Subject: [PATCH 28/30] undo changes to test-infra --- kubernetes/test-infra/eks/main.tf | 2 +- kubernetes/test-infra/eks/node-config/config.tf | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/kubernetes/test-infra/eks/main.tf b/kubernetes/test-infra/eks/main.tf index 96eb9a6253..3058d02e40 100644 --- a/kubernetes/test-infra/eks/main.tf +++ b/kubernetes/test-infra/eks/main.tf @@ -2,7 +2,7 @@ terraform { required_providers { kubernetes = { source = "hashicorp/kubernetes" - version = "9.9.9" + version = "1.13" } aws = { source = "hashicorp/aws" diff --git a/kubernetes/test-infra/eks/node-config/config.tf b/kubernetes/test-infra/eks/node-config/config.tf index 666376ef76..f26010f4a0 100644 --- a/kubernetes/test-infra/eks/node-config/config.tf +++ b/kubernetes/test-infra/eks/node-config/config.tf @@ -9,6 +9,7 @@ data "aws_eks_cluster" "cluster" { provider "kubernetes" { host = var.cluster_endpoint token = data.aws_eks_cluster_auth.cluster.token + load_config_file = false cluster_ca_certificate = base64decode(data.aws_eks_cluster.cluster.certificate_authority.0.data) exec { api_version = "client.authentication.k8s.io/v1alpha1" @@ -195,11 +196,11 @@ resource "kubernetes_deployment" "aws-lb-controller" { timeout_seconds = 10 } resources { - limits = { + limits { cpu = "200m" memory = "500Mi" } - requests = { + requests { cpu = "100m" memory = "200Mi" } From a2c0599d09370f6c3154652d5f51947ba3b156db Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Thu, 21 Jan 2021 12:22:18 -0800 Subject: [PATCH 29/30] Add a sentence about replacing the clusters --- _examples/eks/README.md | 2 +- _examples/gke/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/_examples/eks/README.md b/_examples/eks/README.md index 37255e745a..4e091330b8 100644 --- a/_examples/eks/README.md +++ b/_examples/eks/README.md @@ -1,6 +1,6 @@ # EKS (Amazon Elastic Kubernetes Service) -This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure an EKS cluster. The example config builds the EKS cluster and applies the Kubernetes configurations in a single operation. +This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure an EKS cluster. The example config builds the EKS cluster and applies the Kubernetes configurations in a single operation. This guide will also show you how to make changes to the underlying EKS cluster in such a way that Kuberntes/Helm resources are recreated after the underlying cluster is replaced. You will need the following environment variables to be set: diff --git a/_examples/gke/README.md b/_examples/gke/README.md index 0f66444c53..ae36dfe2ab 100644 --- a/_examples/gke/README.md +++ b/_examples/gke/README.md @@ -1,6 +1,6 @@ # GKE (Google Container Engine) -This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure a GKE cluster. The example config builds the GKE cluster and applies the Kubernetes configurations in a single operation. +This example shows how to use the Terraform Kubernetes Provider and Terraform Helm Provider to configure a GKE cluster. The example config builds the GKE cluster and applies the Kubernetes configurations in a single operation. This guide will also show you how to make changes to the underlying GKE cluster in such a way that Kuberntes/Helm resources are recreated after the underlying cluster is replaced. You will need the following environment variables to be set: From c63ad912a1872641b84abaaf7aaa826afed98902 Mon Sep 17 00:00:00 2001 From: Stef Forrester Date: Thu, 21 Jan 2021 12:23:37 -0800 Subject: [PATCH 30/30] fix error --- _examples/job/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_examples/job/main.tf b/_examples/job/main.tf index 5697b4e3cb..c621c09601 100644 --- a/_examples/job/main.tf +++ b/_examples/job/main.tf @@ -12,7 +12,7 @@ variable "kube_config_file" { } provider "kubernetes" { - config_file = var.kube_config_file + config_path = var.kube_config_file } resource "kubernetes_job" "test" {