initial commit

Initial commit to the repo

---------

Co-authored-by: Eugene Istrati <[email protected]>

.config/functional_tests/pre-entrypoint-helpers.sh

@@ -3,4 +3,8 @@
 ## use this to load any configuration before the functional test 
 ## TIPS: avoid modifying the .project_automation/functional_test/entrypoint.sh
 ## migrate any customization you did on entrypoint.sh to this helper script
-echo "Executing Pre-Entrypoint Helpers"
+echo "Executing Pre-Entrypoint Helpers"
+
+#********** TFC Env Vars *************
+export AWS_DEFAULT_REGION=us-east-1
+export AWS_REGION=us-east-1

.gitignore

@@ -43,4 +43,10 @@ go.mod
 go.sum
 
 # Terraform tests 
-tests/*.auto.tfvars
+tests/*.auto.tfvars
+
+*.tfplan*
+**/builds*
+
+.DS_Store
+**/.DS_Store

.header.md

@@ -1,7 +1,29 @@
-# Terraform Module Project
+# Terraform Module for Amazon Redshift Copy UDF
 
-:no_entry_sign: Do not edit this readme.md file. To learn how to change this content and work with this repository, refer to CONTRIBUTING.md
+This terraform module provides complimentary capabilities to
+[COPY command](https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html)
+by enabling data copy from S3 API compliant storage solutions such as
+[Cloudian](https://github.com/cloudian/cloudian-s3-operator),
+[MinIO](https://github.com/minio/minio), and
+[Weka](https://github.com/weka/csi-wekafs) into Amazon Redshift with
+AWS Lambda UDF (User Defined Function).
 
-## Readme Content
+## Architecture Diagram
 
-This file will contain any instructional information about this module.
+![Architecture Diagram](./docs/diagram.png "Architecture Diagram")
+
+## Usage
+
+```terraform
+module "udf" {
+  source  = "aws-ia/redshift-copy-udf/aws"
+  version = "~> 1.0"
+
+  name        = "redshift-copy-udf"
+  memory_size = 128
+  timeout     = 5
+
+  vpc_subnet_ids     = null # replace with comma separated values
+  security_group_ids = null # replace with comma separated values
+}
+```

CODE_OF_CONDUCT.md

@@ -1,4 +1,5 @@
-## Code of Conduct
+# Code of Conduct
+
 This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
 For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
-[email protected] with any additional questions or comments.
+`[email protected]` with any additional questions or comments.

CONTRIBUTING.md

@@ -8,7 +8,7 @@ For best practices and information on developing with Terraform, see the [I&A Mo
 
 ## Contributing Code
 
-In order to contibute code to this repository, you must submit a *[Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request)*. To do so, you must *[fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo)* this repostiory, make your changes in your forked version and submit a *Pull Request*.
+In order to contribute code to this repository, you must submit a *[Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request)*. To do so, you must *[fork](https://docs.github.com/en/get-started/quickstart/fork-a-repo)* this repository, make your changes in your forked version and submit a *Pull Request*.
 
 ## Writing Documentation
 
@@ -20,7 +20,7 @@ README.md is automatically generated by pulling in content from other files. For
 
 Pull Requests (PRs) submitted against this repository undergo a series of static and functional checks.
 
-> :exclamation: Note: Failures during funtional or static checks will prevent a pull request from being accepted.
+> :exclamation: Note: Failures during functional or static checks will prevent a pull request from being accepted.
 
 It is a best practice to perform these checks locally prior to submitting a pull request.
 
@@ -37,7 +37,7 @@ TIPS: **do not** modify the `./project_automation/{test-name}/entrypoint.sh`, in
 - Checkov
 - Terratest
 
-> :bangbang: The readme.md file will be created after all checks have completed successfuly, it is recommended that you install terraform-docs locally in order to preview your readme.md file prior to publication.
+> :bangbang: The readme.md file will be created after all checks have completed successfully, it is recommended that you install terraform-docs locally in order to preview your readme.md file prior to publication.
 
 ## Install the required tools
 

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright (C) Amazon.com, Inc. or its affiliates. All Rights Reserved.
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -1,31 +1,87 @@
 <!-- BEGIN_TF_DOCS -->
-# Terraform Module Project
+# Terraform Module for Amazon Redshift Copy UDF
 
-:no_entry_sign: Do not edit this readme.md file. To learn how to change this content and work with this repository, refer to CONTRIBUTING.md
+This terraform module provides complimentary capabilities to
+[COPY command](https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html)
+by enabling data copy from S3 API compliant storage solutions such as
+[Cloudian](https://github.com/cloudian/cloudian-s3-operator),
+[MinIO](https://github.com/minio/minio), and
+[Weka](https://github.com/weka/csi-wekafs) into Amazon Redshift with
+AWS Lambda UDF (User Defined Function).
 
-## Readme Content
+## Architecture Diagram
 
-This file will contain any instructional information about this module.
+![Architecture Diagram](./docs/diagram.png "Architecture Diagram")
+
+## Usage
+
+```terraform
+module "udf" {
+  source  = "aws-ia/redshift-copy-udf/aws"
+  version = "~> 1.0"
+
+  name        = "redshift-copy-udf"
+  memory_size = 128
+  timeout     = 5
+
+  vpc_subnet_ids     = null # replace with comma separated values
+  security_group_ids = null # replace with comma separated values
+}
+```
 
 ## Requirements
 
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.0.0 |
+| <a name="requirement_random"></a> [random](#requirement\_random) | >= 3.0.0 |
+
 ## Providers
 
-No providers.
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.0.0 |
+| <a name="provider_random"></a> [random](#provider\_random) | >= 3.0.0 |
 
 ## Modules
 
-No modules.
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_lambda"></a> [lambda](#module\_lambda) | terraform-aws-modules/lambda/aws | ~> 7.0 |
 
 ## Resources
 
-No resources.
+| Name | Type |
+|------|------|
+| [aws_iam_role.redshift](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy.redshift](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource |
+| [random_id.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource |
+| [aws_partition.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
+| [aws_secretsmanager_secret.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/secretsmanager_secret) | data source |
+| [aws_secretsmanager_secret_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/secretsmanager_secret_version) | data source |
 
 ## Inputs
 
-No inputs.
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_memory_size"></a> [memory\_size](#input\_memory\_size) | Lambda UDF memory size | `number` | `128` | no |
+| <a name="input_name"></a> [name](#input\_name) | Lambda UDF function name | `string` | `"redshift-copy-udf"` | no |
+| <a name="input_security_group_ids"></a> [security\_group\_ids](#input\_security\_group\_ids) | Security Group IDs (comma separated values) | `string` | `null` | no |
+| <a name="input_storage_pass"></a> [storage\_pass](#input\_storage\_pass) | Storage Password to Access S3 API Compliant Storage | `string` | `null` | no |
+| <a name="input_storage_secret_arn"></a> [storage\_secret\_arn](#input\_storage\_secret\_arn) | Secrets Manager ARN for S3 API Compliant Storage Credentials | `string` | `null` | no |
+| <a name="input_storage_url"></a> [storage\_url](#input\_storage\_url) | Storage URL to Access S3 API Compliant Storage | `string` | `null` | no |
+| <a name="input_storage_user"></a> [storage\_user](#input\_storage\_user) | Storage Username to Access S3 API Compliant Storage | `string` | `null` | no |
+| <a name="input_timeout"></a> [timeout](#input\_timeout) | Lambda UDF timeout | `number` | `300` | no |
+| <a name="input_vpc_subnet_ids"></a> [vpc\_subnet\_ids](#input\_vpc\_subnet\_ids) | VPC Subnet IDs (comma separated values) | `string` | `null` | no |
 
 ## Outputs
 
-No outputs.
-<!-- END_TF_DOCS -->
+| Name | Description |
+|------|-------------|
+| <a name="output_iam_role_arn"></a> [iam\_role\_arn](#output\_iam\_role\_arn) | IAM Role ARN for Redshift Permissions |
+| <a name="output_iam_role_id"></a> [iam\_role\_id](#output\_iam\_role\_id) | IAM Role ID for Redshift Permissions |
+| <a name="output_iam_role_name"></a> [iam\_role\_name](#output\_iam\_role\_name) | IAM Role Name for Redshift Permissions |
+| <a name="output_lambda_function_arn"></a> [lambda\_function\_arn](#output\_lambda\_function\_arn) | Lambda Function ARN for Redshift UDF |
+| <a name="output_lambda_function_name"></a> [lambda\_function\_name](#output\_lambda\_function\_name) | Lambda Function Name for Redshift UDF |
+<!-- END_TF_DOCS -->

VERSION

@@ -1 +1 @@
-v0.0.0
+v1.0.0

data.tf

@@ -0,0 +1,14 @@
+# Copyright (C) Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+data "aws_partition" "this" {}
+
+data "aws_secretsmanager_secret" "this" {
+  count = local.secret_count
+  arn   = var.storage_secret_arn
+}
+
+data "aws_secretsmanager_secret_version" "this" {
+  count     = local.secret_count
+  secret_id = data.aws_secretsmanager_secret.this[0].id
+}

examples/basic/README.md

@@ -3,27 +3,41 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.14.0 |
-| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.72.0 |
-| <a name="requirement_awscc"></a> [awscc](#requirement\_awscc) | >= 0.11.0 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.0.0 |
 
 ## Providers
 
-No providers.
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.0.0 |
 
 ## Modules
 
-No modules.
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_udf"></a> [udf](#module\_udf) | ../../ | n/a |
 
 ## Resources
 
-No resources.
+| Name | Type |
+|------|------|
+| [aws_availability_zones.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source |
+| [aws_caller_identity.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_partition.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
+| [aws_region.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
 
 ## Inputs
 
 No inputs.
 
 ## Outputs
 
-No outputs.
+| Name | Description |
+|------|-------------|
+| <a name="output_iam_role_arn"></a> [iam\_role\_arn](#output\_iam\_role\_arn) | IAM Role ARN for Redshift Permissions |
+| <a name="output_iam_role_id"></a> [iam\_role\_id](#output\_iam\_role\_id) | IAM Role ID for Redshift Permissions |
+| <a name="output_iam_role_name"></a> [iam\_role\_name](#output\_iam\_role\_name) | IAM Role Name for Redshift Permissions |
+| <a name="output_lambda_function_arn"></a> [lambda\_function\_arn](#output\_lambda\_function\_arn) | Lambda Function ARN for Redshift UDF |
+| <a name="output_lambda_function_name"></a> [lambda\_function\_name](#output\_lambda\_function\_name) | Lambda Function Name for Redshift UDF |
 <!-- END_TF_DOCS -->

examples/basic/data.tf

@@ -0,0 +1,7 @@
+# Copyright (C) Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+data "aws_region" "this" {}
+data "aws_partition" "this" {}
+data "aws_caller_identity" "this" {}
+data "aws_availability_zones" "this" {}

examples/basic/main.tf

@@ -1,5 +1,15 @@
-#####################################################################################
-# Terraform module examples are meant to show an _example_ on how to use a module
-# per use-case. The code below should not be copied directly but referenced in order
-# to build your own root module that invokes this module
-#####################################################################################
+# Copyright (C) Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+module "udf" {
+  source = "../../"
+  # source  = "aws-ia/redshift-copy-udf/aws"
+  # version = "~> 1.0"
+
+  name        = "redshift-copy-udf"
+  memory_size = 128
+  timeout     = 5
+
+  vpc_subnet_ids     = null # replace with comma separated values
+  security_group_ids = null # replace with comma separated values
+}

examples/basic/outputs.tf

@@ -0,0 +1,27 @@
+# Copyright (C) Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+output "iam_role_arn" {
+  description = "IAM Role ARN for Redshift Permissions"
+  value       = module.udf.iam_role_arn
+}
+
+output "iam_role_id" {
+  description = "IAM Role ID for Redshift Permissions"
+  value       = module.udf.iam_role_id
+}
+
+output "iam_role_name" {
+  description = "IAM Role Name for Redshift Permissions"
+  value       = module.udf.iam_role_name
+}
+
+output "lambda_function_arn" {
+  description = "Lambda Function ARN for Redshift UDF"
+  value       = module.udf.lambda_function_arn
+}
+
+output "lambda_function_name" {
+  description = "Lambda Function Name for Redshift UDF"
+  value       = module.udf.lambda_function_name
+}

examples/basic/providers.tf

@@ -1,21 +1,12 @@
+# Copyright (C) Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
 terraform {
-  required_version = ">= 0.14.0"
+  required_version = ">= 1.0.0"
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = ">= 3.72.0"
-    }
-    awscc = {
-      source  = "hashicorp/awscc"
-      version = ">= 0.11.0"
+      version = ">= 4.0.0"
     }
   }
 }
-
-provider "awscc" {
-  user_agent = [{
-    product_name    = "terraform-awscc-"
-    product_version = "0.0.1"
-    comment         = "V1/AWS-D69B4015/<github repo id>"
-  }]
-}