From 6a033ec2702d5eb19951f987307a189c590eb582 Mon Sep 17 00:00:00 2001 From: Godji Fortil Date: Thu, 14 Sep 2023 19:55:36 -0400 Subject: [PATCH 1/7] Revamp the implementation to add support for the Terraform HPCC OSS module Signed-off-by: Godji Fortil --- .gitignore | 2 +- LICENSE | 2 +- README.md | 435 ++++++++++++++++++++++++++++++++---------- data.tf | 5 +- examples/admin.tfvars | 40 ---- examples/providers.tf | 7 + locals.tf | 64 ++++++- main.tf | 54 ++---- output.tf | 15 -- outputs.tf | 3 + providers.tf | 21 -- storage.tf | 107 +++++++++++ variables.tf | 88 +++++++-- versions.tf | 18 ++ 14 files changed, 621 insertions(+), 240 deletions(-) delete mode 100644 examples/admin.tfvars create mode 100644 examples/providers.tf delete mode 100644 output.tf create mode 100644 outputs.tf delete mode 100644 providers.tf create mode 100644 storage.tf create mode 100644 versions.tf diff --git a/.gitignore b/.gitignore index 199e14b..a938d80 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,7 @@ crash.log # version control. # # example.tfvars -admin.tfvars +*.tfvars # Ignore override files as they are usually used to override resources locally and so # are not checked in diff --git a/LICENSE b/LICENSE index 0e6df15..f596f91 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 Godson Fortil +Copyright (c) 2023 HPCC Systems® Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 56a040f..5e439f8 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,16 @@ # Azure - Storage Account for HPCC Systems
-# ** DO NOT USE IN PRODUCTION ** -
-
- - -## Introduction - -This module will deploy a storage account for the HPCC Systems cloud native platform. -
- ## Providers | Name | Version | | ------- | --------- | -| azurerm | >= 2.57.0 | -| random | ~>3.1.0 | +| azurerm | >= 3.63.0 | +| random | >= 3.3.0 | +| azuread | >= 2.42.0 |
-### The `admin` block: +### The `owner` block: This block contains information on the user who is deploying the cluster. This is used as tags and part of some resource names to identify who deployed a given resource and how to contact that user. This block is required. | Name | Description | Type | Default | Required | @@ -31,9 +22,9 @@ This block contains information on the user who is deploying the cluster. This i Usage Example:
- admin = { - name = "Example" - email = "example@hpccdemo.com" + owner = { + name = "demo" + email = "demo@lexisnexisrisk.com" }
@@ -79,105 +70,353 @@ Usage Example:
-### The `tags` argument: -The tag attribute can be used for additional tags. The tags must be key value pairs. This block is optional. +### The `virtual_network` block: +This block imports metadata of a virtual network deployed outside of this project. This block is optional. - | Name | Description | Type | Default | Required | - | ---- | ------------------------- | ----------- | ------- | :------: | - | tags | Additional resource tags. | map(string) | admin | no | -
- -### The `resource_group` block: -This block creates a resource group (like a folder) for your resources. This block is required. - - | Name | Description | Type | Default | Required | - | ----------- | ----------------------------------------------------------------- | ------ | ------- | :------: | - | unique_name | Will concatenate a number at the end of your resource group name. | bool | `true` | yes | - | location | Cloud region in which to deploy the cluster resources. | string | null | yes | + | Name | Description | Type | Default | Required | + | ------------------- | ----------------------------------------------- | ----------- | ------- | :------: | + | name | The name of the private subnet. | string | - | yes | + | resource_group_name | The name of the virtual network resource group. | string | - | yes | + | subnet_ids | The IDs of the subnets to authorize access to. | map(string) | - | yes | + | location | The location of the virtual network | string | - | yes |
Usage Example:
- resource_group = { - unique_name = true - location = "canadacentral" + virtual_network = { + location = "value" + name = "value" + resource_group_name = "value" + subnet_ids = { + "name" = "value" + } }
-### The `storage` block: -This block deploys the HPCC persistent volumes. This block is required. - - | Name | Description | Type | Default | Valid Options | Required | - | ----------------------- | ------------------------------------------------------------------------------------------------------ | ------ | ----------- | ------------------------------------------------------------------------ | :------: | - | access_tier | Defines the access tier for `BlobStorage`, `FileStorage`, `Storage2` accounts. | string | Hot | `Cool`, `Hot` | yes | - | account_kind | Defines the Kind of account. Changing this will destroy your data. | string | `StorageV2` | `BlobStorage`. `BlockBlobStorage`, `FileStorage`, `Storage`, `StorageV2` | yes | - | account_tier | Defines the Tier to use for this storage account. Changing this will destroy your data. | string | `Premium` | `Standard`, `Premium` | yes | - | enable_large_file_share | Enable Large File Share. | bool | `false` | `false`, `true` | yes | - | replication_type | Defines the type of replication to use for this storage account. Changing this will destroy your data. | string | `LRS` | `LRS`, `GRS`, `RAGRS`, `ZRS`, `GZRS`, `RAGZRS` | yes | -
- +### The `storage_accounts` block: +This block deploys the storage accounts for HPCC-Platform data planes. This block is required. + + | Name | Description | Type | Default | Valid Options | Required | + | ------------------------------------ | ----------------------------------------------------------------------------------- | -------------- | ------- | ----------------------- | :------: | + | delete_protection | Should deletion be prevented? | bool | true | `true`, `false` | no | + | prefix_name | The prefix name for the storage account. It must be the storage account object key. | string | - | - | yes | + | storage_type | The storage account type. | string | - | `azurefiles`, `blobnfs` | yes | + | authorized_ip_ranges | Group of IPs to authorize access to the storage account. | Object(string) | `{}` | - | no | + | replication_type | + | subnet_ids | + | file_share_retention_days | + | access_tier | + | account_kind | + | account_tier | + | blob_soft_delete_retention_days | + | container_soft_delete_retention_days | +
+ +#### The `var.storage_accounts.planes` block: + + | Name | Description | Type | Default | Valid Options | Required | + | -------- | ------------------------------------------------------ | ------ | ------- | --------------- | :------: | + | size | The size of the share or HPCC data plane | number | - | - | yes | + | rwmany | The set of permissions for the plane | bool | true | `true`, `false` | yes | + | protocol | The network file sharing protocol to use for the share | string | `NFS` | `SMB`, `NFS` | yes | + | sub_path | The sub path for the HPCC data plane | string | - | - | yes | + | category | The category for the HPCC data plane | string | - | - | yes | + | name | The name of the plane | string | - | - | yes | + | sku | The sku for the plane | string | - | - | no | +
+ Usage Example:
- storage = { - access_tier = "Hot" - account_kind = "StorageV2" - account_tier = "Standard" - account_replication_type = "LRS" - - quotas = { - dali = 3 - data = 2 - dll = 2 - lz = 2 - sasha = 5 + storage_accounts = { + adminsvc1 = { + delete_protection = true //Set to false to allow deletion + prefix_name = "adminsvc1" + storage_type = "azurefiles" + authorized_ip_ranges = {} + replication_type = "ZRS" + subnet_ids = {} + file_share_retention_days = 7 + access_tier = "Hot" + account_kind = "FileStorage" + account_tier = "Premium" + + planes = { + dali = { + category = "dali" + name = "dali" + sub_path = "dalistorage" + size = 100 + sku = "" + rwmany = true + protocol = "nfs" + } + } } - } - -
-## Usage -
    -
  1. - -Clone this repo: `git clone https://github.com/gfortil/terraform-azurerm-hpcc-storage.git`.
    2. - -
  2. Linux and MacOS
    3. -
      -
    1. - -Change directory to terraform-azurerm-hpcc-storage: `cd terraform-azurerm-hpcc-storage`
      2. -
    2. - -Copy examples/admin.tfvars to terraform-azurerm-hpcc-storage: `cp examples/admin.tfvars`
      3. -
    -
  3. Windows OS
    4. -
      -
    1. - -Change directory to terraform-azurerm-hpcc-storage: `cd terraform-azurerm-hpcc-storage`
      2. -
    2. - -Copy examples/admin.tfvars to terraform-azurerm-hpcc-storage: `copy examples/admin.tfvars`
      3. -
    -
  4. - -Open `terraform-azurerm-hpcc-storage/admin.tfvars` file.
    5. -
  5. + adminsvc2 = { + delete_protection = true //Set to false to allow deletion + prefix_name = "adminsvc2" + storage_type = "blobnfs" + authorized_ip_ranges = {} + replication_type = "ZRS" + subnet_ids = {} + blob_soft_delete_retention_days = 7 + container_soft_delete_retention_days = 7 + access_tier = "Hot" + account_kind = "StorageV2" + account_tier = "Standard" + + planes = { + dll = { + category = "dll" + name = "dll" + sub_path = "queries" + size = 100 + sku = "" + rwmany = true + } + + lz = { + category = "lz" + name = "mydropzone" + sub_path = "dropzone" + size = 100 + sku = "" + rwmany = true + } + + sasha = { + category = "sasha" + name = "sasha" + sub_path = "sashastorage" + size = 100 + sku = "" + rwmany = true + } + + debug = { + category = "debug" + name = "debug" + sub_path = "debug" + size = 100 + sku = "" + rwmany = true + } + } + } -Set attributes to your preferred values.
    6. -
  6. + data1 = { + delete_protection = true //Set to false to allow deletion + prefix_name = "data1" + storage_type = "blobnfs" + authorized_ip_ranges = {} + replication_type = "ZRS" + subnet_ids = {} + blob_soft_delete_retention_days = 7 + container_soft_delete_retention_days = 7 + access_tier = "Hot" + account_kind = "StorageV2" + account_tier = "Standard" + + planes = { + data = { + category = "data" + name = "data" + sub_path = "hpcc-data" + size = 100 + sku = "" + rwmany = true + } + } + } -Save `terraform-azurerm-hpcc-storage/admin.tfvars` file.
    7. -
  7. + data2 = { + delete_protection = true //Set to false to allow deletion + prefix_name = "data2" + storage_type = "blobnfs" + authorized_ip_ranges = {} + replication_type = "ZRS" + subnet_ids = {} + blob_soft_delete_retention_days = 7 + container_soft_delete_retention_days = 7 + access_tier = "Hot" + account_kind = "StorageV2" + account_tier = "Standard" + + planes = { + data = { + category = "data" + name = "data" + sub_path = "hpcc-data" + size = 100 + sku = "" + rwmany = true + } + } + } + } +
    -Run `terraform init`. This step is only required before your first `terraform apply`.
    8. -
  8. +## Usage -Run `terraform apply -var-file=admin.tfvars` or `terraform apply -var-file=admin.tfvars -auto-approve`.
    9. -
  9. + module "storage" { + source = "../../../terraform-azurerm-hpcc-storage" -Type `yes` if you didn't pass the flag `-auto-approve`.
    10. -
\ No newline at end of file + owner = { + name = "demo" + email = "demo@lexisnexisrisk.com" + } + disable_naming_conventions = false + metadata = { + project = "hpccplatform" + product_name = "hpccplatform" + business_unit = "commercial" + environment = "sandbox" + market = "us" + product_group = "hpcc" + resource_group_type = "app" + sre_team = "hpccplatform" + subscription_type = "dev" + additional_tags = { "justification" = "testing" } + location = "eastus" # Acceptable values: eastus, centralus + } + virtual_network = { + location = "value" + name = "value" + resource_group_name = "value" + subnet_ids = { + "name" = "value" + } + } + storage_accounts = { + adminsvc1 = { + delete_protection = true //Set to false to allow deletion + prefix_name = "adminsvc1" + storage_type = "azurefiles" + authorized_ip_ranges = {} + replication_type = "ZRS" + subnet_ids = {} + file_share_retention_days = 7 + access_tier = "Hot" + account_kind = "FileStorage" + account_tier = "Premium" + + planes = { + dali = { + category = "dali" + name = "dali" + sub_path = "dalistorage" + size = 100 + sku = "" + rwmany = true + protocol = "nfs" + } + } + } + + adminsvc2 = { + delete_protection = true //Set to false to allow deletion + prefix_name = "adminsvc2" + storage_type = "blobnfs" + authorized_ip_ranges = {} + replication_type = "ZRS" + subnet_ids = {} + blob_soft_delete_retention_days = 7 + container_soft_delete_retention_days = 7 + access_tier = "Hot" + account_kind = "StorageV2" + account_tier = "Standard" + + planes = { + dll = { + category = "dll" + name = "dll" + sub_path = "queries" + size = 100 + sku = "" + rwmany = true + } + + lz = { + category = "lz" + name = "mydropzone" + sub_path = "dropzone" + size = 100 + sku = "" + rwmany = true + } + + sasha = { + category = "sasha" + name = "sasha" + sub_path = "sashastorage" + size = 100 + sku = "" + rwmany = true + } + + debug = { + category = "debug" + name = "debug" + sub_path = "debug" + size = 100 + sku = "" + rwmany = true + } + } + } + + data1 = { + delete_protection = true //Set to false to allow deletion + prefix_name = "data1" + storage_type = "blobnfs" + authorized_ip_ranges = {} + replication_type = "ZRS" + subnet_ids = {} + blob_soft_delete_retention_days = 7 + container_soft_delete_retention_days = 7 + access_tier = "Hot" + account_kind = "StorageV2" + account_tier = "Standard" + + planes = { + data = { + category = "data" + name = "data" + sub_path = "hpcc-data" + size = 100 + sku = "" + rwmany = true + } + } + } + + data2 = { + delete_protection = true //Set to false to allow deletion + prefix_name = "data2" + storage_type = "blobnfs" + authorized_ip_ranges = {} + replication_type = "ZRS" + subnet_ids = {} + blob_soft_delete_retention_days = 7 + container_soft_delete_retention_days = 7 + access_tier = "Hot" + account_kind = "StorageV2" + account_tier = "Standard" + + planes = { + data = { + category = "data" + name = "data" + sub_path = "hpcc-data" + size = 100 + sku = "" + rwmany = true + } + } + } + } + } \ No newline at end of file diff --git a/data.tf b/data.tf index 7d8d741..08682c8 100644 --- a/data.tf +++ b/data.tf @@ -1,6 +1,9 @@ data "http" "host_ip" { - url = "http://ipv4.icanhazip.com" + url = "https://api.ipify.org" } data "azurerm_subscription" "current" { } + +data "azurerm_client_config" "current" { +} diff --git a/examples/admin.tfvars b/examples/admin.tfvars deleted file mode 100644 index 546a2a8..0000000 --- a/examples/admin.tfvars +++ /dev/null @@ -1,40 +0,0 @@ -admin = { - name = "hpccdemo" - email = "hpccdemo@example.com" -} - -metadata = { - project = "hpccdemo" - product_name = "storageaccount" - business_unit = "commercial" - environment = "sandbox" - market = "us" - product_group = "contoso" - resource_group_type = "app" - sre_team = "hpccplatform" - subscription_type = "dev" -} - -tags = { "justification" = "testing" } - -resource_group = { - unique_name = true - location = "eastus2" -} - -storage = { - access_tier = "Hot" - account_kind = "StorageV2" - account_tier = "Standard" - account_replication_type = "LRS" - - quotas = { - dali = 1 - data = 3 - dll = 1 - lz = 1 - sasha = 3 - } -} - -# disable_naming_conventions = false # true will enforce all the arguments of the metadata block above diff --git a/examples/providers.tf b/examples/providers.tf new file mode 100644 index 0000000..e609c7e --- /dev/null +++ b/examples/providers.tf @@ -0,0 +1,7 @@ +provider "azurerm" { + features {} + use_cli = true + storage_use_azuread = true +} + +provider "azuread" {} diff --git a/locals.tf b/locals.tf index 56af3d5..e8bf000 100644 --- a/locals.tf +++ b/locals.tf @@ -3,7 +3,7 @@ locals { { business_unit = var.metadata.business_unit environment = var.metadata.environment - location = var.storage.location + location = var.metadata.location market = var.metadata.market subscription_type = var.metadata.subscription_type }, @@ -12,8 +12,64 @@ locals { var.metadata.resource_group_type != "" ? { resource_group_type = var.metadata.resource_group_type } : {} ) : module.metadata.names - tags = var.disable_naming_conventions ? merge(var.tags, { "admin" = var.admin.name, "email" = var.admin.email, "workspace" = terraform.workspace }) : merge(module.metadata.tags, { "admin" = var.admin.name, "email" = var.admin.email, "workspace" = terraform.workspace }, try(var.tags)) + tags = merge(var.metadata.additional_tags, { "owner" = var.owner.name, "owner_email" = var.owner.email }) - storage_shares = { "dalishare" = var.storage.quotas.dali, "dllsshare" = var.storage.quotas.dll, "sashashare" = var.storage.quotas.sasha, - "datashare" = var.storage.quotas.data, "lzshare" = var.storage.quotas.lz } + location = var.virtual_network.location + + resource_groups = { + storage_accounts = { + tags = { "enclosed resource" = "OSS storage accounts" } + } + } + + subnet_ids = { + for k, v in var.virtual_network.subnet_ids : k => "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${var.virtual_network.resource_group_name}/providers/Microsoft.Network/virtualNetworks/${var.virtual_network.name}/subnets/${v}" + } + + azure_files_pv_protocol = "nfs" + + planes = flatten([ + for k, v in var.storage_accounts : + [ + for x, y in v.planes : { + "container_name" : "${y.name}" + "plane_name" : "${y.name}" + "category" : "${y.category}" + "path" : "${y.sub_path}" + "size" : "${y.size}" + "storage_account" : v.storage_type == "azurefiles" ? "${azurerm_storage_account.azurefiles[v.prefix_name].name}" : "${azurerm_storage_account.blobnfs[v.prefix_name].name}" + "resource_group" : "${module.resource_groups["storage_accounts"].name}" + "storage_type" : "${v.storage_type}" + "protocol" : v.storage_type == "azurefiles" ? "${upper(y.protocol)}" : null + "access_tier" : "${v.access_tier}" + "account_kind" : "${v.account_kind}" + "account_tier" : "${v.account_tier}" + "replication_type" : "${v.replication_type}" + "authorized_ip_ranges" : "${merge(v.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })}" + "subnet_ids" : "${v.subnet_ids}" + "file_share_retention_days" : v.storage_type == "azurefiles" ? "${v.file_share_retention_days}" : null + "prefix_name" : "${v.prefix_name}" + } + ] + ]) + + azurefile_storage_accounts = { + for k, v in var.storage_accounts : k => v if v.storage_type == "azurefiles" + } + + blob_storage_accounts = { + for k, v in var.storage_accounts : k => v if v.storage_type == "blobnfs" + } + + azurefile_planes = { + for k, v in local.planes : k => v if v.storage_type == "azurefiles" + } + + blob_planes = { + for k, v in local.planes : k => v if v.storage_type == "blobnfs" + } + + config = jsonencode({ + "external_storage_config" : "${local.planes}" + }) } diff --git a/main.tf b/main.tf index 959074f..ad90f64 100644 --- a/main.tf +++ b/main.tf @@ -1,13 +1,8 @@ -resource "random_integer" "random" { - min = 1 - max = 3 -} - resource "random_string" "random" { - length = 43 - upper = false - number = false + length = 4 special = false + numeric = false + upper = false } module "subscription" { @@ -25,7 +20,7 @@ module "metadata" { naming_rules = module.naming.yaml market = var.metadata.market - location = var.resource_group.location + location = local.location sre_team = var.metadata.sre_team environment = var.metadata.environment product_name = var.metadata.product_name @@ -33,42 +28,17 @@ module "metadata" { product_group = var.metadata.product_group subscription_type = var.metadata.subscription_type resource_group_type = var.metadata.resource_group_type - subscription_id = data.azurerm_subscription.current.id + subscription_id = module.subscription.output.subscription_id project = var.metadata.project } -module "resource_group" { - source = "github.com/Azure-Terraform/terraform-azurerm-resource-group.git?ref=v2.0.0" - - unique_name = var.resource_group.unique_name - location = var.resource_group.location - names = local.names - tags = local.tags -} - -resource "azurerm_storage_account" "storage_account" { - - name = lower(try("${var.admin.name}hpccsa${random_integer.random.result}", "hpccsa${random_integer.random.result}404")) - resource_group_name = module.resource_group.name - location = module.resource_group.location - account_tier = var.storage.account_tier - account_replication_type = var.storage.account_replication_type - min_tls_version = "TLS1_2" - tags = local.tags -} - -resource "azurerm_storage_share" "storage_shares" { - for_each = local.storage_shares - - name = each.key - storage_account_name = azurerm_storage_account.storage_account.name - quota = each.value +module "resource_groups" { + source = "github.com/Azure-Terraform/terraform-azurerm-resource-group.git?ref=v2.1.0" - acl { - id = random_string.random.result + for_each = local.resource_groups - access_policy { - permissions = "rwdl" - } - } + unique_name = true + location = module.metadata.location + names = module.metadata.names + tags = merge(local.tags, each.value.tags) } diff --git a/output.tf b/output.tf deleted file mode 100644 index de140e8..0000000 --- a/output.tf +++ /dev/null @@ -1,15 +0,0 @@ -output "resource_group_name" { - value = module.resource_group.name -} - -output "subscription_id" { - value = data.azurerm_subscription.current.subscription_id -} - -output "storage_account_name" { - value = azurerm_storage_account.storage_account.name -} - -output "location" { - value = module.resource_group.location -} diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..e7a1e89 --- /dev/null +++ b/outputs.tf @@ -0,0 +1,3 @@ +output "config" { + value = local.config +} diff --git a/providers.tf b/providers.tf deleted file mode 100644 index 616d95c..0000000 --- a/providers.tf +++ /dev/null @@ -1,21 +0,0 @@ -terraform { - required_providers { - random = { - source = "hashicorp/random" - version = "3.1.0" - } - - azurerm = { - source = "hashicorp/azurerm" - version = "2.74.0" - } - } -} - -provider "random" { -} - -provider "azurerm" { - features {} -} - diff --git a/storage.tf b/storage.tf new file mode 100644 index 0000000..2178f10 --- /dev/null +++ b/storage.tf @@ -0,0 +1,107 @@ +resource "azurerm_management_lock" "protect_azurefile_storage_accounts" { + for_each = { + for k, v in var.storage_accounts : k => v if v.storage_type == "azurefiles" && v.delete_protection + } + + name = "protect-storage-${azurerm_storage_account.azurefiles[each.key].name}" + scope = azurerm_storage_account.azurefiles[each.key].id + lock_level = "CanNotDelete" +} + +resource "azurerm_management_lock" "protect_blobnfs_storage_accounts" { + for_each = { + for k, v in var.storage_accounts : k => v if v.storage_type == "blobnfs" && v.delete_protection + } + + name = "protect-storage-${azurerm_storage_account.blobnfs[each.key].name}" + scope = azurerm_storage_account.blobnfs[each.key].id + lock_level = "CanNotDelete" +} + +resource "azurerm_storage_account" "azurefiles" { + for_each = local.azurefile_storage_accounts + + name = "hpcc${each.value.prefix_name}${random_string.random.result}af" + resource_group_name = module.resource_groups["storage_accounts"].name + location = local.location + tags = local.tags + + access_tier = each.value.access_tier + account_kind = each.value.account_kind + account_tier = each.value.account_tier + allow_nested_items_to_be_public = false + min_tls_version = "TLS1_2" + + shared_access_key_enabled = true + + enable_https_traffic_only = false + account_replication_type = each.value.replication_type + + network_rules { + default_action = "Deny" + ip_rules = values(merge(each.value.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) + virtual_network_subnet_ids = values(each.value.subnet_ids) + bypass = ["AzureServices"] + } + share_properties { + retention_policy { + days = each.value.file_share_retention_days + } + } +} + +resource "azurerm_storage_account" "blobnfs" { + for_each = local.blob_storage_accounts + + name = "hpcc${each.value.prefix_name}${random_string.random.result}blob" + resource_group_name = module.resource_groups["storage_accounts"].name + location = local.location + tags = local.tags + + access_tier = each.value.access_tier + account_kind = each.value.account_kind + account_tier = each.value.account_tier + allow_nested_items_to_be_public = false + is_hns_enabled = true + min_tls_version = "TLS1_2" + + shared_access_key_enabled = true + + nfsv3_enabled = true + enable_https_traffic_only = true + account_replication_type = each.value.replication_type + + + network_rules { + default_action = "Deny" + ip_rules = values(merge(each.value.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) + virtual_network_subnet_ids = values(each.value.subnet_ids) + bypass = ["AzureServices"] + } + + blob_properties { + delete_retention_policy { + days = each.value.blob_soft_delete_retention_days + } + container_delete_retention_policy { + days = each.value.container_soft_delete_retention_days + } + } +} + +resource "azurerm_storage_share" "azurefiles" { + for_each = local.azurefile_planes + + name = each.value.container_name + storage_account_name = azurerm_storage_account.azurefiles[each.value.prefix_name].name + quota = each.value.size + enabled_protocol = each.value.protocol +} + +resource "azurerm_storage_container" "blobnfs" { + for_each = local.blob_planes + + name = each.value.container_name + storage_account_name = azurerm_storage_account.blobnfs[each.value.prefix_name].name + container_access_type = "private" +} diff --git a/variables.tf b/variables.tf index e6af1c5..949887b 100644 --- a/variables.tf +++ b/variables.tf @@ -1,9 +1,20 @@ -variable "admin" { +variable "owner" { description = "Information for the user who administers the deployment." type = object({ name = string email = string }) + + validation { + condition = try( + regex("hpccdemo", var.owner.name) != "hpccdemo", true + ) && try( + regex("hpccdemo", var.owner.email) != "hpccdemo", true + ) && try( + regex("@example.com", var.owner.email) != "@example.com", true + ) + error_message = "Your name and email are required in the owner block and must not contain hpccdemo or @example.com." + } } variable "disable_naming_conventions" { @@ -13,27 +24,70 @@ variable "disable_naming_conventions" { } variable "metadata" { - description = "Names" - type = map(string) -} - -variable "resource_group" { - description = "Resource group module variables." - type = any + description = "Metadata module variables." + type = object({ + market = string + sre_team = string + environment = string + product_name = string + business_unit = string + product_group = string + subscription_type = string + resource_group_type = string + project = string + additional_tags = map(string) + location = string + }) default = { - unique_name = true - location = null + business_unit = "" + environment = "" + market = "" + product_group = "" + product_name = "hpcc" + project = "" + resource_group_type = "" + sre_team = "" + subscription_type = "" + additional_tags = {} + location = "" } } -variable "storage" { - description = "HPCC Helm chart variables." - type = any - default = { default = {} } +variable "virtual_network" { + description = "Subnet IDs" + type = object({ + name = string + resource_group_name = string + location = string + subnet_ids = map(string) + }) + default = null } -variable "tags" { - description = "Tags" - type = map(string) +variable "storage_accounts" { + type = map(object({ + delete_protection = optional(bool) + prefix_name = string + storage_type = string + authorized_ip_ranges = optional(map(string)) + replication_type = optional(string, "ZRS") + subnet_ids = optional(map(string)) + file_share_retention_days = optional(number, 7) + access_tier = optional(string, "Hot") + account_kind = string + account_tier = string + blob_soft_delete_retention_days = optional(number, 7) + container_soft_delete_retention_days = optional(number, 7) + + planes = map(object({ + category = string + name = string + sub_path = string + size = number + sku = optional(string) + rwmany = bool + protocol = optional(string, "nfs") + })) + })) } diff --git a/versions.tf b/versions.tf new file mode 100644 index 0000000..706fda3 --- /dev/null +++ b/versions.tf @@ -0,0 +1,18 @@ +terraform { + required_version = ">= 1.4.6" + + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.63.0" + } + random = { + source = "hashicorp/random" + version = ">= 3.3.0" + } + azuread = { + source = "hashicorp/azuread" + version = ">= 2.41.0" + } + } +} From 27977a759bd71752b0c2534937e11ebf18a54152 Mon Sep 17 00:00:00 2001 From: Godji Fortil Date: Wed, 20 Sep 2023 13:19:16 -0400 Subject: [PATCH 2/7] Fix an access denied issue Signed-off-by: Godji Fortil --- locals.tf | 10 +++++----- storage.tf | 4 ++-- variables.tf | 11 +++-------- 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/locals.tf b/locals.tf index e8bf000..fae0cc4 100644 --- a/locals.tf +++ b/locals.tf @@ -14,7 +14,7 @@ locals { tags = merge(var.metadata.additional_tags, { "owner" = var.owner.name, "owner_email" = var.owner.email }) - location = var.virtual_network.location + location = var.metadata.location resource_groups = { storage_accounts = { @@ -22,9 +22,9 @@ locals { } } - subnet_ids = { - for k, v in var.virtual_network.subnet_ids : k => "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${var.virtual_network.resource_group_name}/providers/Microsoft.Network/virtualNetworks/${var.virtual_network.name}/subnets/${v}" - } + # subnet_ids = merge({ + # for k, v in var.virtual_network.subnet_ids : k => "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${var.virtual_network.resource_group_name}/providers/Microsoft.Network/virtualNetworks/${var.virtual_network.name}/subnets/${v}" + # }, var.virtual_network.subnet_ids) azure_files_pv_protocol = "nfs" @@ -46,7 +46,7 @@ locals { "account_tier" : "${v.account_tier}" "replication_type" : "${v.replication_type}" "authorized_ip_ranges" : "${merge(v.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })}" - "subnet_ids" : "${v.subnet_ids}" + "subnet_ids" : "${var.subnet_ids}" "file_share_retention_days" : v.storage_type == "azurefiles" ? "${v.file_share_retention_days}" : null "prefix_name" : "${v.prefix_name}" } diff --git a/storage.tf b/storage.tf index 2178f10..c0509d5 100644 --- a/storage.tf +++ b/storage.tf @@ -40,7 +40,7 @@ resource "azurerm_storage_account" "azurefiles" { network_rules { default_action = "Deny" ip_rules = values(merge(each.value.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) - virtual_network_subnet_ids = values(each.value.subnet_ids) + virtual_network_subnet_ids = var.subnet_ids //values(each.value.subnet_ids) bypass = ["AzureServices"] } share_properties { @@ -75,7 +75,7 @@ resource "azurerm_storage_account" "blobnfs" { network_rules { default_action = "Deny" ip_rules = values(merge(each.value.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) - virtual_network_subnet_ids = values(each.value.subnet_ids) + virtual_network_subnet_ids = var.subnet_ids //values(each.value.subnet_ids) bypass = ["AzureServices"] } diff --git a/variables.tf b/variables.tf index 949887b..dd2f106 100644 --- a/variables.tf +++ b/variables.tf @@ -54,15 +54,10 @@ variable "metadata" { } } -variable "virtual_network" { +variable "subnet_ids" { description = "Subnet IDs" - type = object({ - name = string - resource_group_name = string - location = string - subnet_ids = map(string) - }) - default = null + type = list(string) + default = null } variable "storage_accounts" { From 5773d6d7ca3607e8321b4b7089c21b4cfa318cb9 Mon Sep 17 00:00:00 2001 From: Godji Fortil Date: Tue, 5 Mar 2024 17:23:51 -0500 Subject: [PATCH 3/7] Code improvements Signed-off-by: Godji Fortil --- README.md | 219 +++++++++++++++++++++++++++------------------------ locals.tf | 48 +++++++---- outputs.tf | 20 ++++- storage.tf | 39 +++++---- variables.tf | 23 +++--- versions.tf | 4 - 6 files changed, 198 insertions(+), 155 deletions(-) diff --git a/README.md b/README.md index 5e439f8..5d5d780 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Azure - Storage Account for HPCC Systems +
## Providers @@ -10,7 +11,8 @@ | azuread | >= 2.42.0 |
-### The `owner` block: +### The `owner` block + This block contains information on the user who is deploying the cluster. This is used as tags and part of some resource names to identify who deployed a given resource and how to contact that user. This block is required. | Name | Description | Type | Default | Required | @@ -29,7 +31,8 @@ Usage Example:
-### The `disable_naming_conventions` block: +### The `disable_naming_conventions` block + When set to `true`, this attribute drops the naming conventions set forth by the python module. This attribute is optional. | Name | Description | Type | Default | Required | @@ -37,7 +40,8 @@ When set to `true`, this attribute drops the naming conventions set forth by the | disable_naming_conventions | Disable naming conventions. | bool | `false` | no |
-### The `metadata` block: +### The `metadata` block + TThe arguments in this block are used as tags and part of resources’ names. This block can be omitted when disable_naming_conventions is set to `true`. | Name | Description | Type | Default | Required | @@ -70,7 +74,8 @@ Usage Example:
-### The `virtual_network` block: +### The `virtual_network` block + This block imports metadata of a virtual network deployed outside of this project. This block is optional. | Name | Description | Type | Default | Required | @@ -95,13 +100,14 @@ Usage Example:
-### The `storage_accounts` block: +### The `storage_accounts` block + This block deploys the storage accounts for HPCC-Platform data planes. This block is required. | Name | Description | Type | Default | Valid Options | Required | | ------------------------------------ | ----------------------------------------------------------------------------------- | -------------- | ------- | ----------------------- | :------: | | delete_protection | Should deletion be prevented? | bool | true | `true`, `false` | no | - | prefix_name | The prefix name for the storage account. It must be the storage account object key. | string | - | - | yes | + | storage_account_name_prefix | The prefix name for the storage account. It must be the storage account object key. | string | - | - | yes | | storage_type | The storage account type. | string | - | `azurefiles`, `blobnfs` | yes | | authorized_ip_ranges | Group of IPs to authorize access to the storage account. | Object(string) | `{}` | - | no | | replication_type | @@ -114,7 +120,7 @@ This block deploys the storage accounts for HPCC-Platform data planes. This bloc | container_soft_delete_retention_days |
-#### The `var.storage_accounts.planes` block: +#### The `var.storage_accounts.planes` block | Name | Description | Type | Default | Valid Options | Required | | -------- | ------------------------------------------------------ | ------ | ------- | --------------- | :------: | @@ -123,17 +129,17 @@ This block deploys the storage accounts for HPCC-Platform data planes. This bloc | protocol | The network file sharing protocol to use for the share | string | `NFS` | `SMB`, `NFS` | yes | | sub_path | The sub path for the HPCC data plane | string | - | - | yes | | category | The category for the HPCC data plane | string | - | - | yes | - | name | The name of the plane | string | - | - | yes | + | plane_name | The name of the plane | string | - | - | yes | | sku | The sku for the plane | string | - | - | no |
- + Usage Example:
storage_accounts = { adminsvc1 = { delete_protection = true //Set to false to allow deletion - prefix_name = "adminsvc1" + storage_account_name_prefix = "adminsvc1" storage_type = "azurefiles" authorized_ip_ranges = {} replication_type = "ZRS" @@ -146,7 +152,7 @@ Usage Example: planes = { dali = { category = "dali" - name = "dali" + plane_name = "dali" sub_path = "dalistorage" size = 100 sku = "" @@ -158,7 +164,7 @@ Usage Example: adminsvc2 = { delete_protection = true //Set to false to allow deletion - prefix_name = "adminsvc2" + storage_account_name_prefix = "adminsvc2" storage_type = "blobnfs" authorized_ip_ranges = {} replication_type = "ZRS" @@ -172,7 +178,7 @@ Usage Example: planes = { dll = { category = "dll" - name = "dll" + plane_name = "dll" sub_path = "queries" size = 100 sku = "" @@ -181,7 +187,7 @@ Usage Example: lz = { category = "lz" - name = "mydropzone" + plane_name = "mydropzone" sub_path = "dropzone" size = 100 sku = "" @@ -190,7 +196,7 @@ Usage Example: sasha = { category = "sasha" - name = "sasha" + plane_name = "sasha" sub_path = "sashastorage" size = 100 sku = "" @@ -199,7 +205,7 @@ Usage Example: debug = { category = "debug" - name = "debug" + plane_name = "debug" sub_path = "debug" size = 100 sku = "" @@ -210,7 +216,7 @@ Usage Example: data1 = { delete_protection = true //Set to false to allow deletion - prefix_name = "data1" + storage_account_name_prefix = "data1" storage_type = "blobnfs" authorized_ip_ranges = {} replication_type = "ZRS" @@ -224,7 +230,7 @@ Usage Example: planes = { data = { category = "data" - name = "data" + plane_name = "data" sub_path = "hpcc-data" size = 100 sku = "" @@ -235,7 +241,7 @@ Usage Example: data2 = { delete_protection = true //Set to false to allow deletion - prefix_name = "data2" + storage_account_name_prefix = "data2" storage_type = "blobnfs" authorized_ip_ranges = {} replication_type = "ZRS" @@ -249,7 +255,7 @@ Usage Example: planes = { data = { category = "data" - name = "data" + plane_name = "data" sub_path = "hpcc-data" size = 100 sku = "" @@ -269,7 +275,9 @@ Usage Example: name = "demo" email = "demo@lexisnexisrisk.com" } + disable_naming_conventions = false + metadata = { project = "hpccplatform" product_name = "hpccplatform" @@ -283,6 +291,7 @@ Usage Example: additional_tags = { "justification" = "testing" } location = "eastus" # Acceptable values: eastus, centralus } + virtual_network = { location = "value" name = "value" @@ -291,37 +300,36 @@ Usage Example: "name" = "value" } } - storage_accounts = { + + storage_accounts = { adminsvc1 = { - delete_protection = true //Set to false to allow deletion - prefix_name = "adminsvc1" - storage_type = "azurefiles" - authorized_ip_ranges = {} - replication_type = "ZRS" - subnet_ids = {} - file_share_retention_days = 7 - access_tier = "Hot" - account_kind = "FileStorage" - account_tier = "Premium" + delete_protection = false //Set to false to allow deletion + storage_account_name_prefix = "adminsvc1" + storage_type = "azurefiles" + replication_type = "ZRS" + subnet_ids = {} + file_share_retention_days = 7 + access_tier = "Hot" + account_kind = "FileStorage" + account_tier = "Premium" planes = { - dali = { - category = "dali" - name = "dali" - sub_path = "dalistorage" - size = 100 - sku = "" - rwmany = true - protocol = "nfs" - } + dali = { + category = "dali" + plane_name = "dali" + sub_path = "dalistorage" + size = 100 + sku = "" + rwmany = true + protocol = "smb" + } } } adminsvc2 = { - delete_protection = true //Set to false to allow deletion - prefix_name = "adminsvc2" - storage_type = "blobnfs" - authorized_ip_ranges = {} + delete_protection = false //Set to false to allow deletion + storage_account_name_prefix = "adminsvc2" + storage_type = "blob" replication_type = "ZRS" subnet_ids = {} blob_soft_delete_retention_days = 7 @@ -331,49 +339,48 @@ Usage Example: account_tier = "Standard" planes = { - dll = { - category = "dll" - name = "dll" - sub_path = "queries" - size = 100 - sku = "" - rwmany = true - } - - lz = { - category = "lz" - name = "mydropzone" - sub_path = "dropzone" - size = 100 - sku = "" - rwmany = true - } - - sasha = { - category = "sasha" - name = "sasha" - sub_path = "sashastorage" - size = 100 - sku = "" - rwmany = true - } - - debug = { - category = "debug" - name = "debug" - sub_path = "debug" - size = 100 - sku = "" - rwmany = true - } + dll = { + category = "dll" + plane_name = "dll" + sub_path = "queries" + size = 100 + sku = "" + rwmany = true + } + + lz = { + category = "lz" + plane_name = "mydropzone" + sub_path = "dropzone" + size = 100 + sku = "" + rwmany = true + } + + sasha = { + category = "sasha" + plane_name = "sasha" + sub_path = "sashastorage" + size = 100 + sku = "" + rwmany = true + } + + debug = { + category = "debug" + plane_name = "debug" + sub_path = "debug" + size = 100 + sku = "" + rwmany = true + } } } data1 = { - delete_protection = true //Set to false to allow deletion - prefix_name = "data1" - storage_type = "blobnfs" - authorized_ip_ranges = {} + delete_protection = false //Set to false to allow deletion + storage_account_name_prefix = "data1" + storage_type = "blob" replication_type = "ZRS" subnet_ids = {} blob_soft_delete_retention_days = 7 @@ -383,22 +390,21 @@ Usage Example: account_tier = "Standard" planes = { - data = { - category = "data" - name = "data" - sub_path = "hpcc-data" - size = 100 - sku = "" - rwmany = true - } + data = { + category = "data" + plane_name = "data" + sub_path = "hpcc-data" + size = 100 + sku = "" + rwmany = true + } } } data2 = { - delete_protection = true //Set to false to allow deletion - prefix_name = "data2" - storage_type = "blobnfs" - authorized_ip_ranges = {} + delete_protection = false //Set to false to allow deletion + storage_account_name_prefix = "data2" + storage_type = "blob" replication_type = "ZRS" subnet_ids = {} blob_soft_delete_retention_days = 7 @@ -408,15 +414,20 @@ Usage Example: account_tier = "Standard" planes = { - data = { - category = "data" - name = "data" - sub_path = "hpcc-data" - size = 100 - sku = "" - rwmany = true - } + data = { + category = "data" + plane_name = "data" + sub_path = "hpcc-data" + size = 100 + sku = "" + rwmany = true + } } } } - } \ No newline at end of file + + authorized_ip_ranges = { + "ip_1" = "" + } + + } diff --git a/locals.tf b/locals.tf index fae0cc4..498bf8d 100644 --- a/locals.tf +++ b/locals.tf @@ -22,22 +22,19 @@ locals { } } - # subnet_ids = merge({ - # for k, v in var.virtual_network.subnet_ids : k => "/subscriptions/${data.azurerm_client_config.current.subscription_id}/resourceGroups/${var.virtual_network.resource_group_name}/providers/Microsoft.Network/virtualNetworks/${var.virtual_network.name}/subnets/${v}" - # }, var.virtual_network.subnet_ids) - azure_files_pv_protocol = "nfs" planes = flatten([ for k, v in var.storage_accounts : [ for x, y in v.planes : { - "container_name" : "${y.name}" - "plane_name" : "${y.name}" + "container_name" : "${y.plane_name}" + "plane_name" : "${y.plane_name}" "category" : "${y.category}" "path" : "${y.sub_path}" "size" : "${y.size}" - "storage_account" : v.storage_type == "azurefiles" ? "${azurerm_storage_account.azurefiles[v.prefix_name].name}" : "${azurerm_storage_account.blobnfs[v.prefix_name].name}" + "storage_account_name" : v.storage_type == "azurefiles" ? "${v.storage_account_name_prefix}${random_string.random.result}af" : "${v.storage_account_name_prefix}${random_string.random.result}blob" + "storage_account_name_prefix" : v.storage_account_name_prefix "resource_group" : "${module.resource_groups["storage_accounts"].name}" "storage_type" : "${v.storage_type}" "protocol" : v.storage_type == "azurefiles" ? "${upper(y.protocol)}" : null @@ -45,20 +42,19 @@ locals { "account_kind" : "${v.account_kind}" "account_tier" : "${v.account_tier}" "replication_type" : "${v.replication_type}" - "authorized_ip_ranges" : "${merge(v.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })}" + "authorized_ip_ranges" : "${tomap(merge(var.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body }))}" "subnet_ids" : "${var.subnet_ids}" "file_share_retention_days" : v.storage_type == "azurefiles" ? "${v.file_share_retention_days}" : null - "prefix_name" : "${v.prefix_name}" } ] ]) - azurefile_storage_accounts = { - for k, v in var.storage_accounts : k => v if v.storage_type == "azurefiles" + azurefile_storage_accounts_args = { + for k, v in var.storage_accounts : k => merge({ storage_account_name = "${v.storage_account_name_prefix}${random_string.random.result}af" }, v, { "resource_group_name" = module.resource_groups["storage_accounts"].name }) if v.storage_type == "azurefiles" } - blob_storage_accounts = { - for k, v in var.storage_accounts : k => v if v.storage_type == "blobnfs" + blob_storage_accounts_args = { + for k, v in var.storage_accounts : k => merge({ storage_account_name = "${v.storage_account_name_prefix}${random_string.random.result}blob" }, v, { "resource_group_name" = module.resource_groups["storage_accounts"].name }) if v.storage_type == "blobnfs" } azurefile_planes = { @@ -69,7 +65,27 @@ locals { for k, v in local.planes : k => v if v.storage_type == "blobnfs" } - config = jsonencode({ - "external_storage_config" : "${local.planes}" - }) + azurefile_storage_accounts_attrs = { + for k, v in azurerm_storage_account.azurefiles : k => { + storage_account_name = v.name + resource_group_name = v.resource_group_name + id = v.id + account_replication_type = v.account_replication_type + account_tier = v.account_tier + primary_access_key = v.primary_access_key + primary_location = v.primary_location + } + } + + blob_storage_accounts_attrs = { + for k, v in azurerm_storage_account.blobnfs : k => { + storage_account_name = v.name + resource_group_name = v.resource_group_name + id = v.id + account_replication_type = v.account_replication_type + account_tier = v.account_tier + primary_access_key = v.primary_access_key + primary_location = v.primary_location + } + } } diff --git a/outputs.tf b/outputs.tf index e7a1e89..935a73a 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,3 +1,19 @@ -output "config" { - value = local.config +output "azurefile_storage_accounts_attrs" { + value = local.azurefile_storage_accounts_attrs +} + +output "blob_storage_accounts_attrs" { + value = local.blob_storage_accounts_attrs +} + +output "storage_planes" { + value = local.planes +} + +output "azurefile_planes" { + value = local.azurefile_planes +} + +output "blob_planes" { + value = local.blob_planes } diff --git a/storage.tf b/storage.tf index c0509d5..068124f 100644 --- a/storage.tf +++ b/storage.tf @@ -19,9 +19,9 @@ resource "azurerm_management_lock" "protect_blobnfs_storage_accounts" { } resource "azurerm_storage_account" "azurefiles" { - for_each = local.azurefile_storage_accounts + for_each = local.azurefile_storage_accounts_args - name = "hpcc${each.value.prefix_name}${random_string.random.result}af" + name = each.value.storage_account_name resource_group_name = module.resource_groups["storage_accounts"].name location = local.location tags = local.tags @@ -31,15 +31,13 @@ resource "azurerm_storage_account" "azurefiles" { account_tier = each.value.account_tier allow_nested_items_to_be_public = false min_tls_version = "TLS1_2" - - shared_access_key_enabled = true - - enable_https_traffic_only = false - account_replication_type = each.value.replication_type + shared_access_key_enabled = true + enable_https_traffic_only = false + account_replication_type = each.value.replication_type network_rules { default_action = "Deny" - ip_rules = values(merge(each.value.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) + ip_rules = values(merge(var.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) virtual_network_subnet_ids = var.subnet_ids //values(each.value.subnet_ids) bypass = ["AzureServices"] } @@ -48,12 +46,14 @@ resource "azurerm_storage_account" "azurefiles" { days = each.value.file_share_retention_days } } + + depends_on = [random_string.random] } resource "azurerm_storage_account" "blobnfs" { - for_each = local.blob_storage_accounts + for_each = local.blob_storage_accounts_args - name = "hpcc${each.value.prefix_name}${random_string.random.result}blob" + name = each.value.storage_account_name resource_group_name = module.resource_groups["storage_accounts"].name location = local.location tags = local.tags @@ -64,17 +64,14 @@ resource "azurerm_storage_account" "blobnfs" { allow_nested_items_to_be_public = false is_hns_enabled = true min_tls_version = "TLS1_2" - - shared_access_key_enabled = true - - nfsv3_enabled = true - enable_https_traffic_only = true - account_replication_type = each.value.replication_type - + shared_access_key_enabled = true + nfsv3_enabled = true + enable_https_traffic_only = true + account_replication_type = each.value.replication_type network_rules { default_action = "Deny" - ip_rules = values(merge(each.value.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) + ip_rules = values(merge(var.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) virtual_network_subnet_ids = var.subnet_ids //values(each.value.subnet_ids) bypass = ["AzureServices"] } @@ -87,13 +84,15 @@ resource "azurerm_storage_account" "blobnfs" { days = each.value.container_soft_delete_retention_days } } + + depends_on = [random_string.random] } resource "azurerm_storage_share" "azurefiles" { for_each = local.azurefile_planes name = each.value.container_name - storage_account_name = azurerm_storage_account.azurefiles[each.value.prefix_name].name + storage_account_name = azurerm_storage_account.azurefiles[each.value.storage_account_name_prefix].name quota = each.value.size enabled_protocol = each.value.protocol } @@ -102,6 +101,6 @@ resource "azurerm_storage_container" "blobnfs" { for_each = local.blob_planes name = each.value.container_name - storage_account_name = azurerm_storage_account.blobnfs[each.value.prefix_name].name + storage_account_name = azurerm_storage_account.blobnfs[each.value.storage_account_name_prefix].name container_access_type = "private" } diff --git a/variables.tf b/variables.tf index dd2f106..fa56f77 100644 --- a/variables.tf +++ b/variables.tf @@ -63,9 +63,8 @@ variable "subnet_ids" { variable "storage_accounts" { type = map(object({ delete_protection = optional(bool) - prefix_name = string + storage_account_name_prefix = string storage_type = string - authorized_ip_ranges = optional(map(string)) replication_type = optional(string, "ZRS") subnet_ids = optional(map(string)) file_share_retention_days = optional(number, 7) @@ -76,13 +75,19 @@ variable "storage_accounts" { container_soft_delete_retention_days = optional(number, 7) planes = map(object({ - category = string - name = string - sub_path = string - size = number - sku = optional(string) - rwmany = bool - protocol = optional(string, "nfs") + category = string + plane_name = string + sub_path = string + size = number + sku = optional(string) + rwmany = bool + protocol = optional(string, "nfs") })) })) } + +variable "authorized_ip_ranges" { + description = "Authorized IP ranges" + type = map(string) + default = {} +} \ No newline at end of file diff --git a/versions.tf b/versions.tf index 706fda3..5252b15 100644 --- a/versions.tf +++ b/versions.tf @@ -10,9 +10,5 @@ terraform { source = "hashicorp/random" version = ">= 3.3.0" } - azuread = { - source = "hashicorp/azuread" - version = ">= 2.41.0" - } } } From cdcbd7b460f3c455e266e578f67a7d61fd1f9a2c Mon Sep 17 00:00:00 2001 From: Godji Fortil Date: Mon, 22 Apr 2024 17:09:45 -0400 Subject: [PATCH 4/7] Change the public IP API source Signed-off-by: Godji Fortil --- data.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data.tf b/data.tf index 08682c8..7ba171c 100644 --- a/data.tf +++ b/data.tf @@ -1,5 +1,5 @@ data "http" "host_ip" { - url = "https://api.ipify.org" + url = "https://ifconfig.me" } data "azurerm_subscription" "current" { From 322133761ec567a5f2eab2310a5baf2aec95e7cb Mon Sep 17 00:00:00 2001 From: Godji Fortil Date: Fri, 3 May 2024 16:13:51 -0400 Subject: [PATCH 5/7] Limit access to the provided IPs only, and fix minor fixes. Signed-off-by: Godji Fortil --- data.tf | 2 +- storage.tf | 4 ++-- variables.tf | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/data.tf b/data.tf index 7ba171c..989dc47 100644 --- a/data.tf +++ b/data.tf @@ -1,5 +1,5 @@ data "http" "host_ip" { - url = "https://ifconfig.me" + url = "https://ipconfig.org" } data "azurerm_subscription" "current" { diff --git a/storage.tf b/storage.tf index 068124f..3d2f272 100644 --- a/storage.tf +++ b/storage.tf @@ -37,7 +37,7 @@ resource "azurerm_storage_account" "azurefiles" { network_rules { default_action = "Deny" - ip_rules = values(merge(var.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) + ip_rules = var.use_authorized_ip_ranges_only ? values(var.authorized_ip_ranges) : values(merge(var.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) virtual_network_subnet_ids = var.subnet_ids //values(each.value.subnet_ids) bypass = ["AzureServices"] } @@ -71,7 +71,7 @@ resource "azurerm_storage_account" "blobnfs" { network_rules { default_action = "Deny" - ip_rules = values(merge(var.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) + ip_rules = var.use_authorized_ip_ranges_only ? values(var.authorized_ip_ranges) : values(merge(var.authorized_ip_ranges, { host_ip = data.http.host_ip.response_body })) virtual_network_subnet_ids = var.subnet_ids //values(each.value.subnet_ids) bypass = ["AzureServices"] } diff --git a/variables.tf b/variables.tf index fa56f77..615fb9c 100644 --- a/variables.tf +++ b/variables.tf @@ -90,4 +90,10 @@ variable "authorized_ip_ranges" { description = "Authorized IP ranges" type = map(string) default = {} +} + +variable "use_authorized_ip_ranges_only" { + description = "Should Terraform only use the provided IPs for authorization?" + type = bool + default = false } \ No newline at end of file From 8f4f3057f363445aed9752f4afba2d87952afcad Mon Sep 17 00:00:00 2001 From: Godji Fortil Date: Mon, 5 Aug 2024 10:10:58 -0400 Subject: [PATCH 6/7] Change API URL for data.http This fixes an issue with reading the public IP where the entire html body is read instead of the IP value alone. --- data.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data.tf b/data.tf index 989dc47..08682c8 100644 --- a/data.tf +++ b/data.tf @@ -1,5 +1,5 @@ data "http" "host_ip" { - url = "https://ipconfig.org" + url = "https://api.ipify.org" } data "azurerm_subscription" "current" { From 7a7248d09f72ff8e8b4be0f9ebe94d1ea9283bd3 Mon Sep 17 00:00:00 2001 From: Godji Fortil Date: Fri, 23 Aug 2024 03:15:05 -0400 Subject: [PATCH 7/7] Upgrade to azurerm provider 4.0 Signed-off-by: Godji Fortil --- storage.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage.tf b/storage.tf index 3d2f272..d15344a 100644 --- a/storage.tf +++ b/storage.tf @@ -32,7 +32,7 @@ resource "azurerm_storage_account" "azurefiles" { allow_nested_items_to_be_public = false min_tls_version = "TLS1_2" shared_access_key_enabled = true - enable_https_traffic_only = false + https_traffic_only_enabled = false account_replication_type = each.value.replication_type network_rules { @@ -66,7 +66,7 @@ resource "azurerm_storage_account" "blobnfs" { min_tls_version = "TLS1_2" shared_access_key_enabled = true nfsv3_enabled = true - enable_https_traffic_only = true + https_traffic_only_enabled = true account_replication_type = each.value.replication_type network_rules {