From f73494cb1507bdb67b377ccd8be47eed6c51f84f Mon Sep 17 00:00:00 2001 From: David Szabo <38033309+daszabo@users.noreply.github.com> Date: Wed, 9 Oct 2024 14:20:11 +0200 Subject: [PATCH] Express onboarding fixes (#48) --- aws/quickstart.sh | 28 ++++++++++++++++++++++------ azure/main.tf | 2 +- azure/quickstart.sh | 30 ++++++++++++++++++++++++------ azure/variables.tf | 8 ++++++++ gcp/quickstart.sh | 28 ++++++++++++++++++++++------ 5 files changed, 77 insertions(+), 19 deletions(-) diff --git a/aws/quickstart.sh b/aws/quickstart.sh index 223ba06..e3c68ee 100644 --- a/aws/quickstart.sh +++ b/aws/quickstart.sh @@ -19,11 +19,12 @@ export TF_VAR_env_prefix="${2:-""}" export ACCOUNT_ID="${3:-""}" export CDP_REGION="${4:-"us-west-1"}" export TF_VAR_deployment_template="${5:-"semi-private"}" -export TF_VAR_ingress_extra_cidrs_and_ports="${6:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"}" +export TF_VAR_ingress_extra_cidrs_and_ports=${6:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"} export TF_VAR_env_tags='{"deploy_tool": "express-tf", "env_prefix": "'"$2"'"}' export TF_VAR_create_vpc_endpoints="false" export TF_VAR_environment_async_creation="true" export TF_VAR_datalake_async_creation="true" +export TF_VAR_datalake_scale="LIGHT_DUTY" # Save TF variables to file output_file="variables.sh" @@ -33,15 +34,27 @@ export TF_VAR_aws_region="${TF_VAR_aws_region}" export TF_VAR_env_prefix="${TF_VAR_env_prefix}" export ACCOUNT_ID="${ACCOUNT_ID}" export TF_VAR_deployment_template="${TF_VAR_deployment_template}" -export TF_VAR_env_tags="${TF_VAR_env_tags}" +export TF_VAR_env_tags='${TF_VAR_env_tags}' export TF_VAR_create_vpc_endpoints="${TF_VAR_create_vpc_endpoints}" export TF_VAR_environment_async_creation="${TF_VAR_environment_async_creation}" export TF_VAR_datalake_async_creation="${TF_VAR_datalake_async_creation}" -export TF_VAR_ingress_extra_cidrs_and_ports="${TF_VAR_ingress_extra_cidrs_and_ports}" +export TF_VAR_ingress_extra_cidrs_and_ports='${TF_VAR_ingress_extra_cidrs_and_ports}' +export TF_VAR_datalake_scale="${TF_VAR_datalake_scale}" +EOF + +destroy_file="destroy.sh" + +cat < $destroy_file +cd cdp-tf-quickstarts/aws + +source ${HOME}/variables.sh + +${HOME}/terraform destroy -auto-approve EOF # Make the file executable chmod +x $output_file +chmod +x $destroy_file # Install Terraform curl -fsSL https://releases.hashicorp.com/terraform/1.7.1/terraform_1.7.1_linux_amd64.zip -o terraform.zip @@ -49,15 +62,18 @@ unzip -o terraform.zip -d ${HOME} rm terraform.zip # Checkout CDP Quickstart Repository -git clone --branch v0.7.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git +git clone --branch v0.8.3 https://github.com/cloudera-labs/cdp-tf-quickstarts.git cd cdp-tf-quickstarts/aws # Install CDP CLI and Log In pip install cdpcli config_file="${HOME}/.cdp/config" -region_config="cdp_region = ${CDP_REGION}" -echo "$region_config" >> "$config_file" +mkdir -p "$(dirname "$config_file")" +cat < $config_file +[default] +cdp_region = ${CDP_REGION} +EOF cdp login --account-id "${ACCOUNT_ID}" --use-device-code diff --git a/azure/main.tf b/azure/main.tf index 31867cd..239591b 100644 --- a/azure/main.tf +++ b/azure/main.tf @@ -92,7 +92,7 @@ module "cdp_deploy" { datalake_async_creation = var.datalake_async_creation # From pre-reqs module output - azure_subscription_id = module.cdp_azure_prereqs.azure_subscription_id + azure_subscription_id = var.azure_subscription_id == null ? module.cdp_azure_prereqs.azure_subscription_id : var.azure_subscription_id azure_tenant_id = module.cdp_azure_prereqs.azure_tenant_id azure_resource_group_name = module.cdp_azure_prereqs.azure_resource_group_name diff --git a/azure/quickstart.sh b/azure/quickstart.sh index 1f74c5b..62dd33c 100644 --- a/azure/quickstart.sh +++ b/azure/quickstart.sh @@ -19,10 +19,12 @@ export TF_VAR_env_prefix="${2:-""}" export ACCOUNT_ID="${3:-""}" export CDP_REGION="${4:-"us-west-1"}" export TF_VAR_deployment_template="${5:-"semi-private"}" -export TF_VAR_ingress_extra_cidrs_and_ports="${6:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"}" +export TF_VAR_ingress_extra_cidrs_and_ports=${6:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"} export TF_VAR_env_tags='{"deploy_tool": "express-tf", "env_prefix": "'"$2"'"}' export TF_VAR_environment_async_creation="true" export TF_VAR_datalake_async_creation="true" +export TF_VAR_datalake_scale="LIGHT_DUTY" +export TF_VAR_azure_subscription_id="$(az account show --query id --output tsv)" # Save TF variables to file output_file="variables.sh" @@ -32,25 +34,41 @@ export TF_VAR_azure_region="${TF_VAR_azure_region}" export TF_VAR_env_prefix="${TF_VAR_env_prefix}" export ACCOUNT_ID="${ACCOUNT_ID}" export TF_VAR_deployment_template="${TF_VAR_deployment_template}" -export TF_VAR_env_tags="${TF_VAR_env_tags}" +export TF_VAR_env_tags='${TF_VAR_env_tags}' export TF_VAR_environment_async_creation="${TF_VAR_environment_async_creation}" export TF_VAR_datalake_async_creation="${TF_VAR_datalake_async_creation}" -export TF_VAR_ingress_extra_cidrs_and_ports="${TF_VAR_ingress_extra_cidrs_and_ports}" +export TF_VAR_ingress_extra_cidrs_and_ports='${TF_VAR_ingress_extra_cidrs_and_ports}' +export TF_VAR_datalake_scale="${TF_VAR_datalake_scale}" +export TF_VAR_azure_subscription_id="${TF_VAR_azure_subscription_id}" +EOF + +destroy_file="destroy.sh" + +cat < $destroy_file +cd cdp-tf-quickstarts/azure + +source ${HOME}/variables.sh + +${HOME}/terraform destroy -auto-approve EOF # Make the file executable chmod +x $output_file +chmod +x $destroy_file # Checkout CDP Quickstart Repository -git clone --branch v0.7.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git +git clone --branch v0.8.3 https://github.com/cloudera-labs/cdp-tf-quickstarts.git cd cdp-tf-quickstarts/azure # Install CDP CLI and Log In pip install cdpcli config_file="${HOME}/.cdp/config" -region_config="cdp_region = ${CDP_REGION}" -echo "$region_config" >> "$config_file" +mkdir -p "$(dirname "$config_file")" +cat < $config_file +[default] +cdp_region = ${CDP_REGION} +EOF cdp login --account-id "${ACCOUNT_ID}" --use-device-code diff --git a/azure/variables.tf b/azure/variables.tf index 79de355..6fd9f92 100644 --- a/azure/variables.tf +++ b/azure/variables.tf @@ -105,6 +105,14 @@ variable "datalake_recipes" { default = null } +variable "azure_subscription_id" { + type = string + + description = "The subscription where the environment should be deployed to" + + default = null +} + # ------- Network Resources ------- variable "ingress_extra_cidrs_and_ports" { type = object({ diff --git a/gcp/quickstart.sh b/gcp/quickstart.sh index f0c6f94..efb5839 100644 --- a/gcp/quickstart.sh +++ b/gcp/quickstart.sh @@ -20,10 +20,11 @@ export TF_VAR_env_prefix="${3:-""}" export ACCOUNT_ID="${4:-""}" export CDP_REGION="${5:-"us-west-1"}" export TF_VAR_deployment_template="${6:-"semi-private"}" -export TF_VAR_ingress_extra_cidrs_and_ports="${7:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"}" +export TF_VAR_ingress_extra_cidrs_and_ports=${6:-"{ cidrs = [\"0.0.0.0/0\"], ports = [443, 22] }"} export TF_VAR_env_tags='{"deploy_tool": "express-tf", "env_prefix": "'"$2"'"}' export TF_VAR_environment_async_creation="true" export TF_VAR_datalake_async_creation="true" +export TF_VAR_datalake_scale="LIGHT_DUTY" # Save TF variables to file output_file="variables.sh" @@ -31,28 +32,43 @@ output_file="variables.sh" cat < $output_file export TF_VAR_gcp_project="${TF_VAR_gcp_project}" export TF_VAR_gcp_region="${TF_VAR_gcp_region}" -export TF_VAR_env_prefix="${TF_VAR_env_prefix}" +export TF_VAR_env_prefix='${TF_VAR_env_prefix}' export ACCOUNT_ID="${ACCOUNT_ID}" export TF_VAR_deployment_template="${TF_VAR_deployment_template}" export TF_VAR_env_tags="${TF_VAR_env_tags}" export TF_VAR_environment_async_creation="${TF_VAR_environment_async_creation}" export TF_VAR_datalake_async_creation="${TF_VAR_datalake_async_creation}" -export TF_VAR_ingress_extra_cidrs_and_ports="${TF_VAR_ingress_extra_cidrs_and_ports}" +export TF_VAR_ingress_extra_cidrs_and_ports='${TF_VAR_ingress_extra_cidrs_and_ports}' +export TF_VAR_datalake_scale="${TF_VAR_datalake_scale}" +EOF + +destroy_file="destroy.sh" + +cat < $destroy_file +cd cdp-tf-quickstarts/gcp + +source ${HOME}/variables.sh + +${HOME}/terraform destroy -auto-approve EOF # Make the file executable chmod +x $output_file +chmod +x $destroy_file # Checkout CDP Quickstart Repository -git clone --branch v0.7.2 https://github.com/cloudera-labs/cdp-tf-quickstarts.git +git clone --branch v0.8.3 https://github.com/cloudera-labs/cdp-tf-quickstarts.git cd cdp-tf-quickstarts/gcp # Install CDP CLI and Log In pip install cdpcli config_file="${HOME}/.cdp/config" -region_config="cdp_region = ${CDP_REGION}" -echo "$region_config" >> "$config_file" +mkdir -p "$(dirname "$config_file")" +cat < $config_file +[default] +cdp_region = ${CDP_REGION} +EOF cdp login --account-id "${ACCOUNT_ID}" --use-device-code