From 07cd664ef7ca271b8e8c77941c234be17be587d6 Mon Sep 17 00:00:00 2001 From: Siva Guruvareddiar Date: Fri, 29 Mar 2024 16:18:10 -0500 Subject: [PATCH] superset changes --- analytics/terraform/superset-on-eks/README.md | 98 +++++++++---------- analytics/terraform/superset-on-eks/addons.tf | 54 ++-------- .../terraform/superset-on-eks/ingres.yaml | 2 +- analytics/terraform/superset-on-eks/main.tf | 60 ++++-------- .../{ => superset-yamls}/pv.yaml | 0 .../terraform/superset-on-eks/variables.tf | 4 +- .../data-analytics/superset-on-eks.md | 8 +- 7 files changed, 84 insertions(+), 142 deletions(-) rename analytics/terraform/superset-on-eks/{ => superset-yamls}/pv.yaml (100%) diff --git a/analytics/terraform/superset-on-eks/README.md b/analytics/terraform/superset-on-eks/README.md index a028dfb98..89efc0bcd 100644 --- a/analytics/terraform/superset-on-eks/README.md +++ b/analytics/terraform/superset-on-eks/README.md @@ -1,54 +1,52 @@ -# Apache Superset on EKS -This repository contains Terraform configuration to deploy Apache Superset on AWS infrastructure. - -## Architecture -Terraform will create following AWS infrastructure for Superset: -- VPC -- Subnets (Public/Private) -- EC2 Instance for Superset -- Security Groups -- IAM Roles and Policies - -## Pre-requisites -- AWS account -- AWS CLI installed and configured -- Terraform v0.12+ installed -- Helm - -## Usage -- Clone this repository -- Update variables.tf with your AWS details -- Initialize Terraform - ``` - terraform init - ``` -- Review execution plan - ``` - terraform plan - ``` - -- Provision infrastructure - ``` - terraform apply - ``` - -- Access the Superset web UI at http://PUBLIC_IP - -- Default credentials are admin/admin - -- Destroy infrastructure when done - ``` - terraform destroy - ``` +## Requirements -## Resources -Following resources will be created by Terraform: + For security reasons, ALB is deployed as internal one and it can be changed to internet-facing during the deployment, if needed. +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | 5.36.0 | +| [helm](#provider\_helm) | 2.12.1 | +| [kubernetes](#provider\_kubernetes) | 2.25.2 | +| [null](#provider\_null) | 3.2.2 | -- VPC with public and private subnets -- Postgres database (superset-db) -- EC2 instance (Apache Superset) -- Security Groups for DB & EC2 instances -- IAM Roles & Policies +## Modules +| Name | Source | Version | +|------|--------|---------| +| [ebs\_csi\_driver\_irsa](#module\_ebs\_csi\_driver\_irsa) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.20 | +| [eks](#module\_eks) | terraform-aws-modules/eks/aws | ~> 19.15 | +| [eks\_blueprints\_addons](#module\_eks\_blueprints\_addons) | aws-ia/eks-blueprints-addons/aws | ~> 1.2 | +| [lb\_role](#module\_lb\_role) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | 5.37.1 | +| [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 | + +## Resources -Review the resources section in main.tf file +| Name | Type | +|------|------| +| [helm_release.alb_controller](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.superset](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_ingress_class_v1.aws_alb](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/ingress_class_v1) | resource | +| [kubernetes_ingress_v1.superset](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/ingress_v1) | resource | +| [kubernetes_namespace.superset](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [kubernetes_service_account.service_account](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account) | resource | +| [null_resource.add_superset_repo](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [null_resource.helm_update_repos](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | +| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [eks\_cluster\_version](#input\_eks\_cluster\_version) | EKS Cluster version | `string` | `"1.28"` | no | +| [name](#input\_name) | Name of the VPC and EKS Cluster | `string` | `"superset-on-eks"` | no | +| [region](#input\_region) | Region | `string` | `"us-east-1"` | no | +| [secondary\_cidr\_blocks](#input\_secondary\_cidr\_blocks) | Secondary CIDR blocks to be attached to VPC | `list(string)` | 
[
  "100.64.0.0/16"
]
| no | +| [vpc\_cidr](#input\_vpc\_cidr) | VPC CIDR. This should be a valid private (RFC 1918) CIDR range | `string` | `"10.1.0.0/21"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [configure\_kubectl](#output\_configure\_kubectl) | Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig | +| [superset\_url](#output\_superset\_url) | Configure kubectl: Once the kubeconfig is configured as above, use the below command to get the Superset URL | diff --git a/analytics/terraform/superset-on-eks/addons.tf b/analytics/terraform/superset-on-eks/addons.tf index 0f988d0ba..b081c7c58 100755 --- a/analytics/terraform/superset-on-eks/addons.tf +++ b/analytics/terraform/superset-on-eks/addons.tf @@ -58,52 +58,13 @@ resource "kubernetes_service_account" "service_account" { } -resource "helm_release" "alb_controller" { - name = "aws-load-balancer-controller" - repository = "https://aws.github.io/eks-charts" - chart = "aws-load-balancer-controller" - namespace = "kube-system" - depends_on = [ - kubernetes_service_account.service_account - ] - - set { - name = "region" - value = var.region - } - - set { - name = "vpcId" - value = module.vpc.vpc_id - } - - set { - name = "image.repository" - value = "602401143452.dkr.ecr.${var.region}.amazonaws.com/amazon/aws-load-balancer-controller" - } - - set { - name = "serviceAccount.create" - value = "false" - } - - set { - name = "serviceAccount.name" - value = "aws-load-balancer-controller" - } - - set { - name = "clusterName" - value = var.name - } -} resource "kubernetes_ingress_v1" "superset" { metadata { name = "superset-ingress3" namespace = "superset" annotations = { - "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "alb.ingress.kubernetes.io/scheme" = "internal-facing" "alb.ingress.kubernetes.io/target-type" = "ip" } } @@ -142,10 +103,10 @@ module "eks_blueprints_addons" { # Amazon EKS Managed Add-ons #--------------------------------------- eks_addons = { - aws-ebs-csi-driver = { - most_recent = true - service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn - } + # aws-ebs-csi-driver = { + # most_recent = true + # service_account_role_arn = module.ebs_csi_driver_irsa.iam_role_arn + # } coredns = { preserve = true } @@ -157,7 +118,8 @@ module "eks_blueprints_addons" { } } enable_aws_load_balancer_controller = true - - + aws_load_balancer_controller = { + chart_version = "1.5.4" + } tags = local.tags } diff --git a/analytics/terraform/superset-on-eks/ingres.yaml b/analytics/terraform/superset-on-eks/ingres.yaml index 6335497bc..68dd5e347 100644 --- a/analytics/terraform/superset-on-eks/ingres.yaml +++ b/analytics/terraform/superset-on-eks/ingres.yaml @@ -4,7 +4,7 @@ metadata: namespace: superset name: ingress annotations: - alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/scheme: internal-facing alb.ingress.kubernetes.io/target-type: ip spec: ingressClassName: alb diff --git a/analytics/terraform/superset-on-eks/main.tf b/analytics/terraform/superset-on-eks/main.tf index 9ec25829e..3e429b07f 100755 --- a/analytics/terraform/superset-on-eks/main.tf +++ b/analytics/terraform/superset-on-eks/main.tf @@ -10,8 +10,28 @@ locals { } } +terraform { + required_version = "~> 1.3.0" - + required_providers { + null = { + source = "hashicorp/null" + version = "3.1.1" + } + helm = { + source = "hashicorp/helm" + version = "2.5.1" + } + aws = { + source = "hashicorp/aws" + version = "~> 3.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.11.0" + } + } +} #--------------------------------------------------------------- # EKS Cluster #--------------------------------------------------------------- @@ -163,41 +183,3 @@ resource "helm_release" "superset" { ] } - - - -# # Allow traffic from ALB to worker nodes -# resource "aws_security_group_rule" "allow_alb" { -# type = "ingress" -# from_port = 8088 -# to_port = 8088 -# protocol = "tcp" -# security_group_id = aws_eks_cluster.eks.worker_security_group_id -# source_security_group_id = aws_security_group.alb.id -# } - -# # Allow traffic from worker nodes to pods -# resource "aws_security_group_rule" "allow_workers_to_pods" { -# type = "egress" -# from_port = 8088 -# to_port = 8088 -# protocol = "tcp" -# security_group_id = aws_eks_cluster.eks.worker_security_group_id -# source_security_group_id = aws_eks_cluster.eks.worker_security_group_id -# } - -# Allow pod ingress from ALB -# resource "kubernetes_network_policy" "allow_alb" { -# metadata { -# name = "allow-alb" -# } - -# spec { -# pod_selector {} -# ingress { -# from { -# security_group = aws_security_group.alb.id -# } -# } -# } -# } diff --git a/analytics/terraform/superset-on-eks/pv.yaml b/analytics/terraform/superset-on-eks/superset-yamls/pv.yaml similarity index 100% rename from analytics/terraform/superset-on-eks/pv.yaml rename to analytics/terraform/superset-on-eks/superset-yamls/pv.yaml diff --git a/analytics/terraform/superset-on-eks/variables.tf b/analytics/terraform/superset-on-eks/variables.tf index 07a6672b1..327b3d97d 100755 --- a/analytics/terraform/superset-on-eks/variables.tf +++ b/analytics/terraform/superset-on-eks/variables.tf @@ -7,12 +7,12 @@ variable "name" { variable "region" { description = "Region" type = string - default = "us-east-1" + default = "us-west-2" } variable "eks_cluster_version" { description = "EKS Cluster version" - default = "1.28" + default = "1.29" type = string } diff --git a/website/docs/blueprints/data-analytics/superset-on-eks.md b/website/docs/blueprints/data-analytics/superset-on-eks.md index b2666b267..880badd4e 100644 --- a/website/docs/blueprints/data-analytics/superset-on-eks.md +++ b/website/docs/blueprints/data-analytics/superset-on-eks.md @@ -25,7 +25,7 @@ The blueprint performs the following to deploy Superset on EKS: - Create a new VPC with public and private subnets - Provision an EKS cluster control plane and managed worker nodes -- Ceate an Amazon EBS file system and access point +- Create an Amazon EBS file system and access point - Build Docker images and push to Amazon ECR - Install Superset and services on EKS via Helm chart - Expose Superset UI through a load balancer @@ -59,18 +59,18 @@ Navigate into one of the example directories and run `install.sh` script ```bash cd data-on-eks/analytics/terraform/superset-on-eks chmod +x install.sh -./install.sh +./install.sh ``` or simply ```bash -terraform init +terraform init terraform apply --auto-approve ``` ### Verify Deployment -After the deployment completes, we can access the Superset UI . For demo purpose, this blueprint creates the Ingress object for the Superset FrontEnd UI with public LoadBalancer. +After the deployment completes, we can access the Superset UI . For demo purpose, this blueprint creates the Ingress object for the Superset FrontEnd UI with public LoadBalancer. You may find the URL to the Superset frontend from the output superset_url, or by running kubectl command below: