diff --git a/.header.md b/.header.md index 13f04a3..9a9d379 100644 --- a/.header.md +++ b/.header.md @@ -10,7 +10,7 @@ AWS Lambda UDF (User Defined Function). ## Architecture Diagram -![Architecture Diagram](./docs/diagram.png "Architecture Diagram") +![Architecture Diagram](./docs/architecture-diagram.png "Architecture Diagram") ## Usage @@ -25,5 +25,11 @@ module "udf" { vpc_subnet_ids = null # replace with comma separated values security_group_ids = null # replace with comma separated values + + storage_url = null # replace with storage url + storage_user = null # replace with storage user + storage_pass = null # replace with storage pass + storage_token = null # replace with storage token (optional) + storage_secret_arn = null # replace with secrets manager arn (optional) } ``` diff --git a/README.md b/README.md index 53eb31b..fb86873 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ AWS Lambda UDF (User Defined Function). ## Architecture Diagram -![Architecture Diagram](./docs/diagram.png "Architecture Diagram") +![Architecture Diagram](./docs/architecture-diagram.png "Architecture Diagram") ## Usage @@ -26,6 +26,12 @@ module "udf" { vpc_subnet_ids = null # replace with comma separated values security_group_ids = null # replace with comma separated values + + storage_url = null # replace with storage url + storage_user = null # replace with storage user + storage_pass = null # replace with storage pass + storage_token = null # replace with storage token (optional) + storage_secret_arn = null # replace with secrets manager arn (optional) } ``` @@ -69,7 +75,8 @@ module "udf" { | [name](#input\_name) | Lambda UDF function name | `string` | `"redshift-copy-udf"` | no | | [security\_group\_ids](#input\_security\_group\_ids) | Security Group IDs (comma separated values) | `string` | `null` | no | | [storage\_pass](#input\_storage\_pass) | Storage Password to Access S3 API Compliant Storage | `string` | `null` | no | -| [storage\_secret\_arn](#input\_storage\_secret\_arn) | Secrets Manager ARN for S3 API Compliant Storage Credentials | `string` | `null` | no | +| [storage\_secret\_arn](#input\_storage\_secret\_arn) | Secrets Manager ARN Holding Credentials to Access S3 API Compliant Storage (Optional) | `string` | `null` | no | +| [storage\_token](#input\_storage\_token) | Storage Token to Access S3 API Compliant Storage (Optional) | `string` | `null` | no | | [storage\_url](#input\_storage\_url) | Storage URL to Access S3 API Compliant Storage | `string` | `null` | no | | [storage\_user](#input\_storage\_user) | Storage Username to Access S3 API Compliant Storage | `string` | `null` | no | | [timeout](#input\_timeout) | Lambda UDF timeout | `number` | `300` | no | diff --git a/VERSION b/VERSION index 0ec25f7..b18d465 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.0.0 +v1.0.1 diff --git a/docs/diagram.png b/docs/architecture-diagram.png similarity index 57% rename from docs/diagram.png rename to docs/architecture-diagram.png index 1e0e87a..992202e 100644 Binary files a/docs/diagram.png and b/docs/architecture-diagram.png differ diff --git a/docs/minio-diagram.png b/docs/minio-diagram.png new file mode 100644 index 0000000..6f8c9b6 Binary files /dev/null and b/docs/minio-diagram.png differ diff --git a/examples/basic/.header.md b/examples/basic/.header.md index e69de29..572ce44 100644 --- a/examples/basic/.header.md +++ b/examples/basic/.header.md @@ -0,0 +1,39 @@ +# Basic Example + +This terraform module provides complimentary capabilities to +[COPY command](https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html) +by enabling data copy from S3 API compliant storage solutions such as +[Cloudian](https://github.com/cloudian/cloudian-s3-operator), +[MinIO](https://github.com/minio/minio), and +[Weka](https://github.com/weka/csi-wekafs) into Amazon Redshift with +AWS Lambda UDF (User Defined Function). + +## Architecture Diagram + +![Architecture Diagram](../../docs/architecture-diagram.png "Architecture Diagram") + +## Usage + +* Initialize terraform configs and modules + +```sh +terraform init +``` + +* Review the resources to be created by terraform + +```sh +terraform plan +``` + +* Apply the changes reviewed in the previous step + +```sh +terraform apply +``` + +* When you need to clean up resources + +```sh +terraform destroy +``` diff --git a/examples/basic/README.md b/examples/basic/README.md index af8e74e..c44cc8d 100644 --- a/examples/basic/README.md +++ b/examples/basic/README.md @@ -1,4 +1,44 @@ +# Basic Example + +This terraform module provides complimentary capabilities to +[COPY command](https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html) +by enabling data copy from S3 API compliant storage solutions such as +[Cloudian](https://github.com/cloudian/cloudian-s3-operator), +[MinIO](https://github.com/minio/minio), and +[Weka](https://github.com/weka/csi-wekafs) into Amazon Redshift with +AWS Lambda UDF (User Defined Function). + +## Architecture Diagram + +![Architecture Diagram](../../docs/architecture-diagram.png "Architecture Diagram") + +## Usage + +* Initialize terraform configs and modules + +```sh +terraform init +``` + +* Review the resources to be created by terraform + +```sh +terraform plan +``` + +* Apply the changes reviewed in the previous step + +```sh +terraform apply +``` + +* When you need to clean up resources + +```sh +terraform destroy +``` + ## Requirements | Name | Version | diff --git a/examples/minio/.header.md b/examples/minio/.header.md index e69de29..0c5e6e6 100644 --- a/examples/minio/.header.md +++ b/examples/minio/.header.md @@ -0,0 +1,39 @@ +# MinIO Example + +This terraform module provides complimentary capabilities to +[COPY command](https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html) +by enabling data copy from S3 API compliant storage solutions such as +[Cloudian](https://github.com/cloudian/cloudian-s3-operator), +[MinIO](https://github.com/minio/minio), and +[Weka](https://github.com/weka/csi-wekafs) into Amazon Redshift with +AWS Lambda UDF (User Defined Function). + +## Architecture Diagram + +![Architecture Diagram](../../docs/minio-diagram.png "Architecture Diagram") + +## Usage + +* Initialize terraform configs and modules + +```sh +terraform init +``` + +* Review the resources to be created by terraform + +```sh +terraform plan +``` + +* Apply the changes reviewed in the previous step + +```sh +terraform apply +``` + +* When you need to clean up resources + +```sh +terraform destroy +``` diff --git a/examples/minio/README.md b/examples/minio/README.md index 42383a5..31cd85c 100644 --- a/examples/minio/README.md +++ b/examples/minio/README.md @@ -1,4 +1,44 @@ +# MinIO Example + +This terraform module provides complimentary capabilities to +[COPY command](https://docs.aws.amazon.com/redshift/latest/dg/r_COPY.html) +by enabling data copy from S3 API compliant storage solutions such as +[Cloudian](https://github.com/cloudian/cloudian-s3-operator), +[MinIO](https://github.com/minio/minio), and +[Weka](https://github.com/weka/csi-wekafs) into Amazon Redshift with +AWS Lambda UDF (User Defined Function). + +## Architecture Diagram + +![Architecture Diagram](../../docs/minio-diagram.png "Architecture Diagram") + +## Usage + +* Initialize terraform configs and modules + +```sh +terraform init +``` + +* Review the resources to be created by terraform + +```sh +terraform plan +``` + +* Apply the changes reviewed in the previous step + +```sh +terraform apply +``` + +* When you need to clean up resources + +```sh +terraform destroy +``` + ## Requirements | Name | Version | @@ -44,7 +84,7 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [cidr](#input\_cidr) | This is the CIDR block for your EKS cluster | `string` | `"10.0.0.0/16"` | no | -| [k8s](#input\_k8s) | This is the version of your EKS cluster | `string` | `"1.29"` | no | +| [k8s](#input\_k8s) | This is the version of your EKS cluster | `string` | `"1.31"` | no | | [name](#input\_name) | This is the name of your EKS cluster | `string` | `"redshift-minio-demo"` | no | ## Outputs @@ -56,5 +96,5 @@ | [iam\_role\_name](#output\_iam\_role\_name) | IAM Role Name for Redshift Permissions | | [lambda\_function\_arn](#output\_lambda\_function\_arn) | Lambda Function ARN for Redshift UDF | | [lambda\_function\_name](#output\_lambda\_function\_name) | Lambda Function Name for Redshift UDF | -| [storage\_instructions](#output\_storage\_instructions) | n/a | +| [storage\_instructions](#output\_storage\_instructions) | Instructions to install the MinIO storage solution | \ No newline at end of file diff --git a/examples/minio/minio-tenant.yaml b/examples/minio/minio-tenant.yaml index e06ef6e..1be0162 100644 --- a/examples/minio/minio-tenant.yaml +++ b/examples/minio/minio-tenant.yaml @@ -9,7 +9,7 @@ metadata: apiVersion: v1 kind: Secret metadata: - name: storage-configuration + name: minio-config namespace: minio-tenant type: Opaque stringData: @@ -25,14 +25,14 @@ metadata: name: minio namespace: minio-tenant spec: - serviceMetadata: - minioServiceAnnotations: - service.beta.kubernetes.io/aws-load-balancer-security-groups: "sg-0a4cecff434c80e6e" + # serviceMetadata: + # minioServiceAnnotations: + # service.beta.kubernetes.io/aws-load-balancer-security-groups: "" exposeServices: console: true minio: true configuration: - name: storage-configuration + name: minio-config image: quay.io/minio/minio:RELEASE.2023-05-27T05-56-19Z pools: - servers: 4 diff --git a/examples/minio/outputs.tf b/examples/minio/outputs.tf index 283342c..7bdc0d7 100644 --- a/examples/minio/outputs.tf +++ b/examples/minio/outputs.tf @@ -27,8 +27,9 @@ output "lambda_function_name" { } output "storage_instructions" { - value = < len(lines): - print(f"[ERROR] line number out of range: {nr_line} > {len(lines)}") - raise MyException("line number out of range") - - rec = [] - if nr_line < 0: - rec.append(str(len(lines))) - elif lines: - for line in lines[nr_line:nr_line + nr_rec]: - rec.append(line) - - if len(rec) != nr_rec: - print(f"[ERROR] number of records mismatch: {len(rec)} != {nr_rec}") - raise MyException("number of records mismatch") - - result["success"] = True - result["results"] = rec - - except MyException as e: - result["error_msg"] = str(e) - print(f"[ERROR] {str(e)}") - - except Exception as e: - result["error_msg"] = str(e) - print(f"[ERROR] {str(e)}") - exc_type, exc_obj, exc_tb = sys.exc_info() - print(f"[ERROR] exc_info: {exc_type}, {exc_tb.tb_lineno}") - - return json.dumps(result) diff --git a/lambda_cfn/output.csv b/lambda_cfn/output.csv deleted file mode 100644 index e69de29..0000000 diff --git a/locals.tf b/locals.tf index cb14028..9ddcdf0 100644 --- a/locals.tf +++ b/locals.tf @@ -11,9 +11,10 @@ locals { local.secret_count > 0 ? jsondecode(data.aws_secretsmanager_secret_version.this[0].secret_string) : { - STORAGE_URL = var.storage_url - STORAGE_USER = var.storage_user - STORAGE_PASS = var.storage_pass + STORAGE_URL = var.storage_url + STORAGE_USER = var.storage_user + STORAGE_PASS = var.storage_pass + STORAGE_TOKEN = var.storage_token } ) } diff --git a/variables.tf b/variables.tf index cc92b31..44b159f 100644 --- a/variables.tf +++ b/variables.tf @@ -31,12 +31,6 @@ variable "vpc_subnet_ids" { description = "VPC Subnet IDs (comma separated values)" } -variable "storage_secret_arn" { - type = string - default = null - description = "Secrets Manager ARN for S3 API Compliant Storage Credentials" -} - variable "storage_url" { type = string default = null @@ -47,10 +41,25 @@ variable "storage_user" { type = string default = null description = "Storage Username to Access S3 API Compliant Storage" + sensitive = true } variable "storage_pass" { type = string default = null description = "Storage Password to Access S3 API Compliant Storage" + sensitive = true +} + +variable "storage_token" { + type = string + default = null + description = "Storage Token to Access S3 API Compliant Storage (Optional)" + sensitive = true +} + +variable "storage_secret_arn" { + type = string + default = null + description = "Secrets Manager ARN Holding Credentials to Access S3 API Compliant Storage (Optional)" }