From 305232cac41db05055d4c6d05ca53b3c17cb4321 Mon Sep 17 00:00:00 2001 From: owl-king Date: Wed, 7 Aug 2024 21:21:42 +0700 Subject: [PATCH 01/25] Allow to enable lifecycle for snapshot bucket, load balancer logs bucket and rds snapshot bucket (#111) Default: - enable lifecycle - 7 days for fullnode snapshot, 31 days for logs and rds snapshot --- indexer/s3_bucket.tf | 43 +++++++++++++++++++++++++++++++++++++++++++ indexer/variables.tf | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/indexer/s3_bucket.tf b/indexer/s3_bucket.tf index e0b9636b..7c2516cf 100644 --- a/indexer/s3_bucket.tf +++ b/indexer/s3_bucket.tf @@ -9,6 +9,20 @@ resource "aws_s3_bucket" "load_balancer" { } } +resource "aws_s3_bucket_lifecycle_configuration" "load_balancer" { + count = var.enable_s3_load_balancer_logs_lifecycle ? 1 : 0 + bucket = aws_s3_bucket.load_balancer.id + + rule { + id = "expire-old-logs" + status = "Enabled" + + expiration { + days = var.s3_load_balancer_logs_expiration_days + } + } +} + # TODO: refactor snapshotting full node into a separate module # AWS S3 bucket to store all Indexer full node snapshots resource "aws_s3_bucket" "indexer_full_node_snapshots" { @@ -22,6 +36,21 @@ resource "aws_s3_bucket" "indexer_full_node_snapshots" { } } +resource "aws_s3_bucket_lifecycle_configuration" "indexer_full_node_snapshots" { + count = var.enable_s3_snapshot_lifecycle ? 1 : 0 + bucket = aws_s3_bucket.indexer_full_node_snapshots.id + + rule { + id = "expire-old-snapshots" + status = "Enabled" + + expiration { + days = var.s3_snapshot_expiration_days + } + } +} + + # Enable S3 bucket metrics to be sent to Datadog for monitoring resource "aws_s3_bucket_metric" "indexer_full_node_snapshots" { bucket = aws_s3_bucket.indexer_full_node_snapshots.id @@ -64,3 +93,17 @@ resource "aws_s3_bucket" "athena_rds_snapshots" { Environment = var.environment } } + +resource "aws_s3_bucket_lifecycle_configuration" "athena_rds_snapshots" { + count = var.enable_s3_rds_snapshot_lifecycle ? 1 : 0 + bucket = aws_s3_bucket.athena_rds_snapshots.id + + rule { + id = "expire-old-snapshots" + status = "Enabled" + + expiration { + days = var.s3_rds_snapshot_expiration_days + } + } +} diff --git a/indexer/variables.tf b/indexer/variables.tf index 37f1ed6d..52df4816 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -472,3 +472,39 @@ variable "image_count" { description = "Number of images to store for ECR, defaults to 100." default = 100 } + +variable "enable_s3_snapshot_lifecycle" { + type = bool + description = "Enables S3 lifecycle on snapshot bucket. Default is true" + default = true +} + +variable "s3_snapshot_expiration_days" { + type = number + description = "Number of days to store fullnode snapshot on S3, defaults to 7." + default = 7 +} + +variable "enable_s3_rds_snapshot_lifecycle" { + type = bool + description = "Enables S3 lifecycle on rds snapshot bucket. Default is true" + default = true +} + +variable "s3_rds_snapshot_expiration_days" { + type = number + description = "Number of days to store rds snapshot on S3, defaults to 31." + default = 31 +} + +variable "enable_s3_load_balancer_logs_lifecycle" { + type = bool + description = "Enables S3 lifecycle on snapshot bucket. Default is true" + default = true +} + +variable "s3_load_balancer_logs_expiration_days" { + type = number + description = "Number of days to store load balancer logs on S3, defaults to 31." + default = 31 +} From 42ddef9f28f3cd8986c41c5dd50873a8fa7cd657 Mon Sep 17 00:00:00 2001 From: owl-king Date: Thu, 8 Aug 2024 15:35:02 +0700 Subject: [PATCH 02/25] Allow to enable/disable multi_az option on master node and remove read_replica_2 instance on dev* (#113) --- indexer/rds.tf | 3 ++- indexer/route53.tf | 3 ++- indexer/variables.tf | 12 ++++++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/indexer/rds.tf b/indexer/rds.tf index 72a7383e..bfa4316e 100644 --- a/indexer/rds.tf +++ b/indexer/rds.tf @@ -215,7 +215,7 @@ resource "aws_db_instance" "main" { performance_insights_enabled = true performance_insights_retention_period = 31 auto_minor_version_upgrade = false - multi_az = true + multi_az = var.enable_rds_main_multiaz tags = { Name = local.aws_db_instance_main_name @@ -250,6 +250,7 @@ resource "aws_db_instance" "read_replica" { # Read replica 2 resource "aws_db_instance" "read_replica_2" { + count = var.create_read_replica_2 ? 1 : 0 identifier = "${local.aws_db_instance_main_name}-read-replica-2" instance_class = var.rds_db_instance_class # engine, engine_version, name, username, db_subnet_group_name, allocated_storage do not have to diff --git a/indexer/route53.tf b/indexer/route53.tf index 60b2ad26..6adac3f0 100644 --- a/indexer/route53.tf +++ b/indexer/route53.tf @@ -19,11 +19,12 @@ resource "aws_route53_record" "read_replica_1" { } resource "aws_route53_record" "read_replica_2" { + count = var.create_read_replica_2 ? 1 : 0 zone_id = aws_route53_zone.main.zone_id name = "postgres-main-rr.dydx-indexer.private" type = "CNAME" ttl = "30" - records = ["${aws_db_instance.read_replica_2.address}"] + records = ["${aws_db_instance.read_replica_2[count.index].address}"] weighted_routing_policy { weight = 1 } diff --git a/indexer/variables.tf b/indexer/variables.tf index 52df4816..f464fa3e 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -508,3 +508,15 @@ variable "s3_load_balancer_logs_expiration_days" { description = "Number of days to store load balancer logs on S3, defaults to 31." default = 31 } + +variable "create_read_replica_2" { + description = "Create read replia 2 or not. Default: true" + type = bool + default = true +} + +variable "enable_rds_main_multiaz" { + description = "Enable RDS main instance. Default: true" + type = bool + default = true +} From b25fdc05c8e476361821d324de6fde8bf53011d3 Mon Sep 17 00:00:00 2001 From: owl-king Date: Thu, 8 Aug 2024 15:37:21 +0700 Subject: [PATCH 03/25] Reduce fullnode, fullnode_snapshot, and fullnode_backup storage to 1TB (#114) --- indexer/backup_full_node_ap_northeast_1.tf | 2 ++ indexer/full_node_ap_northeast_1.tf | 2 ++ indexer/variables.tf | 6 ++++++ 3 files changed, 10 insertions(+) diff --git a/indexer/backup_full_node_ap_northeast_1.tf b/indexer/backup_full_node_ap_northeast_1.tf index d95b16fa..d7a07430 100644 --- a/indexer/backup_full_node_ap_northeast_1.tf +++ b/indexer/backup_full_node_ap_northeast_1.tf @@ -37,6 +37,8 @@ module "backup_full_node_ap_northeast_1" { use_persistent_docker_volume = var.full_node_use_persistent_docker_volume + root_block_device_size = var.full_node_root_block_device_size + providers = { aws = aws.ap_northeast_1 } diff --git a/indexer/full_node_ap_northeast_1.tf b/indexer/full_node_ap_northeast_1.tf index 5b39959e..29b34c67 100644 --- a/indexer/full_node_ap_northeast_1.tf +++ b/indexer/full_node_ap_northeast_1.tf @@ -36,6 +36,8 @@ module "full_node_ap_northeast_1" { use_persistent_docker_volume = var.full_node_use_persistent_docker_volume + root_block_device_size = var.full_node_root_block_device_size + providers = { aws = aws.ap_northeast_1 } diff --git a/indexer/variables.tf b/indexer/variables.tf index f464fa3e..079b47c5 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -162,6 +162,12 @@ variable "full_node_container_chain_home" { description = "Full-node's home directory for the chain. Used to boot up the chain, and configure the `cmd` in ECS" } +variable "full_node_root_block_device_size" { + type = number + description = "Size of root block device in gigabytes" + default = 3000 +} + variable "snapshot_full_node_container_chain_home" { type = string description = "Snapshot full-node's home directory for the chain. Used to boot up the chain, and configure the `cmd` in ECS" From 0588f12ebf3e34e94360651b490bf45c3491ffd8 Mon Sep 17 00:00:00 2001 From: owl-king Date: Thu, 8 Aug 2024 15:40:48 +0700 Subject: [PATCH 04/25] Change root_block_device_delete_on_termination = true to avoid unattached volumes (#115) --- indexer/backup_full_node_ap_northeast_1.tf | 3 ++- indexer/full_node_ap_northeast_1.tf | 3 ++- indexer/snapshot_full_node_ap_northeast_1.tf | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/indexer/backup_full_node_ap_northeast_1.tf b/indexer/backup_full_node_ap_northeast_1.tf index d7a07430..74679033 100644 --- a/indexer/backup_full_node_ap_northeast_1.tf +++ b/indexer/backup_full_node_ap_northeast_1.tf @@ -37,7 +37,8 @@ module "backup_full_node_ap_northeast_1" { use_persistent_docker_volume = var.full_node_use_persistent_docker_volume - root_block_device_size = var.full_node_root_block_device_size + root_block_device_size = var.full_node_root_block_device_size + root_block_device_delete_on_termination = true providers = { aws = aws.ap_northeast_1 diff --git a/indexer/full_node_ap_northeast_1.tf b/indexer/full_node_ap_northeast_1.tf index 29b34c67..470f5fbc 100644 --- a/indexer/full_node_ap_northeast_1.tf +++ b/indexer/full_node_ap_northeast_1.tf @@ -36,7 +36,8 @@ module "full_node_ap_northeast_1" { use_persistent_docker_volume = var.full_node_use_persistent_docker_volume - root_block_device_size = var.full_node_root_block_device_size + root_block_device_size = var.full_node_root_block_device_size + root_block_device_delete_on_termination = true providers = { aws = aws.ap_northeast_1 diff --git a/indexer/snapshot_full_node_ap_northeast_1.tf b/indexer/snapshot_full_node_ap_northeast_1.tf index 832c905c..8d7ff590 100644 --- a/indexer/snapshot_full_node_ap_northeast_1.tf +++ b/indexer/snapshot_full_node_ap_northeast_1.tf @@ -42,7 +42,8 @@ module "full_node_snapshot_ap_northeast_1" { datadog_env = "snapshot-${var.environment}" - root_block_device_size = var.full_node_snapshot_ebs_volume_size + root_block_device_size = var.full_node_snapshot_ebs_volume_size + root_block_device_delete_on_termination = true entry_point = [ "sh", From 1bab89a244b1c428f12aa7c1e9fd83651c65d234 Mon Sep 17 00:00:00 2001 From: owl-king Date: Thu, 8 Aug 2024 15:45:56 +0700 Subject: [PATCH 05/25] Set assign_public_ip = false as all the tasks are in private subnet (#116) --- indexer/ecs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/ecs.tf b/indexer/ecs.tf index b7a172c9..7ffc821b 100644 --- a/indexer/ecs.tf +++ b/indexer/ecs.tf @@ -50,7 +50,7 @@ resource "aws_ecs_service" "main" { aws_subnet.private_subnets[subnet_name].id ] : [for subnet in aws_subnet.private_subnets : subnet.id] security_groups = [aws_security_group.services[each.key].id] - assign_public_ip = true + assign_public_ip = false } dynamic "load_balancer" { From 14fd9a4fcf77b08c71dc55f1bb6ee84fac0bf43f Mon Sep 17 00:00:00 2001 From: owl-king Date: Thu, 8 Aug 2024 16:09:37 +0700 Subject: [PATCH 06/25] Reduce msk storage of dev env to 500 (#117) --- indexer/msk.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/msk.tf b/indexer/msk.tf index 69112991..c0aca667 100644 --- a/indexer/msk.tf +++ b/indexer/msk.tf @@ -36,7 +36,7 @@ resource "aws_msk_cluster" "main" { instance_type = var.msk_instance_type storage_info { ebs_storage_info { - volume_size = var.environment == "mainnet" ? 2000 : 1000 # in GB + volume_size = var.environment == "mainnet" ? 2000 : (contains(["dev", "dev2", "dev3", "dev4", "dev5"], var.environment) ? 500 : 1000) # in GB } } client_subnets = [ From 59c3b26b325351ebee323cafbe4f8cd89e096f5f Mon Sep 17 00:00:00 2001 From: owl-king Date: Thu, 8 Aug 2024 17:06:01 +0700 Subject: [PATCH 07/25] Reduce socks memory to 8192 (#118) --- indexer/locals.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/locals.tf b/indexer/locals.tf index 1169f232..d0e57fff 100644 --- a/indexer/locals.tf +++ b/indexer/locals.tf @@ -98,7 +98,7 @@ locals { }, "${local.service_names["socks"]}" : { ecs_desired_count : 5, - task_definition_memory : 20480, + task_definition_memory : 8192, task_definition_cpu : 4096, is_public_facing : true, ports : [8080, 8000], From 74e8fbb2cbe0f089c21edb41c7ce9d9fee7ba665 Mon Sep 17 00:00:00 2001 From: owl-king Date: Thu, 8 Aug 2024 21:51:35 +0700 Subject: [PATCH 08/25] Allow to changes ecs tasks architecture, lambda funciton. Default value: (#119) X86_64 to be ready with existing docker images --- indexer/backup_full_node_ap_northeast_1.tf | 1 + indexer/ecs.tf | 1 + indexer/full_node_ap_northeast_1.tf | 1 + indexer/lambda.tf | 2 +- indexer/snapshot_full_node_ap_northeast_1.tf | 1 + indexer/variables.tf | 39 ++++++++++++++++++++ modules/validator/ecs.tf | 5 +++ modules/validator/variables.tf | 13 +++++++ 8 files changed, 62 insertions(+), 1 deletion(-) diff --git a/indexer/backup_full_node_ap_northeast_1.tf b/indexer/backup_full_node_ap_northeast_1.tf index 74679033..6a59bce7 100644 --- a/indexer/backup_full_node_ap_northeast_1.tf +++ b/indexer/backup_full_node_ap_northeast_1.tf @@ -39,6 +39,7 @@ module "backup_full_node_ap_northeast_1" { root_block_device_size = var.full_node_root_block_device_size root_block_device_delete_on_termination = true + ecs_task_cpu_architecture = var.fullnode_ecs_task_cpu_architecture providers = { aws = aws.ap_northeast_1 diff --git a/indexer/ecs.tf b/indexer/ecs.tf index 7ffc821b..0743def9 100644 --- a/indexer/ecs.tf +++ b/indexer/ecs.tf @@ -162,6 +162,7 @@ resource "aws_ecs_task_definition" "main" { runtime_platform { operating_system_family = "LINUX" + cpu_architecture = var.indexer_ecs_task_cpu_architecture } tags = { diff --git a/indexer/full_node_ap_northeast_1.tf b/indexer/full_node_ap_northeast_1.tf index 470f5fbc..84c04301 100644 --- a/indexer/full_node_ap_northeast_1.tf +++ b/indexer/full_node_ap_northeast_1.tf @@ -38,6 +38,7 @@ module "full_node_ap_northeast_1" { root_block_device_size = var.full_node_root_block_device_size root_block_device_delete_on_termination = true + ecs_task_cpu_architecture = var.fullnode_ecs_task_cpu_architecture providers = { aws = aws.ap_northeast_1 diff --git a/indexer/lambda.tf b/indexer/lambda.tf index 32f67ec2..db71fa2a 100644 --- a/indexer/lambda.tf +++ b/indexer/lambda.tf @@ -12,7 +12,7 @@ resource "aws_lambda_function" "main" { package_type = "Image" function_name = "${each.key}_lambda_function" role = aws_iam_role.lambda_services[each.key].arn - architectures = ["x86_64"] + architectures = [lower(var.lambda_cpu_architecture)] timeout = 120 environment { diff --git a/indexer/snapshot_full_node_ap_northeast_1.tf b/indexer/snapshot_full_node_ap_northeast_1.tf index 8d7ff590..92c1af4d 100644 --- a/indexer/snapshot_full_node_ap_northeast_1.tf +++ b/indexer/snapshot_full_node_ap_northeast_1.tf @@ -44,6 +44,7 @@ module "full_node_snapshot_ap_northeast_1" { root_block_device_size = var.full_node_snapshot_ebs_volume_size root_block_device_delete_on_termination = true + ecs_task_cpu_architecture = var.fullnode_ecs_task_cpu_architecture entry_point = [ "sh", diff --git a/indexer/variables.tf b/indexer/variables.tf index 079b47c5..85786696 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -526,3 +526,42 @@ variable "enable_rds_main_multiaz" { type = bool default = true } + +variable "indexer_ecs_task_cpu_architecture" { + type = string + description = "Type of ecs cpu architecture. Accept: X86_64 or ARM64" + default = "X86_64" + validation { + condition = contains( + ["X86_64", "ARM64"], + var.indexer_ecs_task_cpu_architecture + ) + error_message = "Err: invalid environment. Must be one of {X86_64 | ARM64}." + } +} + +variable "fullnode_ecs_task_cpu_architecture" { + type = string + description = "Type of ecs cpu architecture. Accept: X86_64 or ARM64" + default = "X86_64" + validation { + condition = contains( + ["X86_64", "ARM64"], + var.fullnode_ecs_task_cpu_architecture + ) + error_message = "Err: invalid environment. Must be one of {X86_64 | ARM64}." + } +} + +variable "lambda_cpu_architecture" { + type = string + description = "Type of lambda cpu architecture. Accept: X86_64 or ARM64" + default = "X86_64" + validation { + condition = contains( + ["X86_64", "ARM64"], + var.lambda_cpu_architecture + ) + error_message = "Err: invalid environment. Must be one of {X86_64 | ARM64}." + } +} diff --git a/modules/validator/ecs.tf b/modules/validator/ecs.tf index 1cd2810f..f62c9b45 100644 --- a/modules/validator/ecs.tf +++ b/modules/validator/ecs.tf @@ -230,6 +230,11 @@ resource "aws_ecs_task_definition" "main" { } } + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = var.ecs_task_cpu_architecture + } + tags = { Name = "${var.environment}-${var.name}-task" Environment = var.environment diff --git a/modules/validator/variables.tf b/modules/validator/variables.tf index 1c02d851..e07c25de 100644 --- a/modules/validator/variables.tf +++ b/modules/validator/variables.tf @@ -266,3 +266,16 @@ variable "dd_site" { default = "datadoghq.com" description = "The site that the datadog agent will send data to" } + +variable "ecs_task_cpu_architecture" { + type = string + description = "Type of ecs cpu architecture. Accept: X86_64 or ARM64" + default = "X86_64" + validation { + condition = contains( + ["X86_64", "ARM64"], + var.ecs_task_cpu_architecture + ) + error_message = "Err: invalid environment. Must be one of {X86_64 | ARM64}." + } +} From ea1ba0f608345964848c80298e76d4d38d226d46 Mon Sep 17 00:00:00 2001 From: owl-king Date: Fri, 9 Aug 2024 09:44:16 +0700 Subject: [PATCH 09/25] Set default retetion for s3_rds_snapshot and load_balancer_logs to 14 (#121) --- indexer/variables.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/indexer/variables.tf b/indexer/variables.tf index 85786696..cc4f6f25 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -499,8 +499,8 @@ variable "enable_s3_rds_snapshot_lifecycle" { variable "s3_rds_snapshot_expiration_days" { type = number - description = "Number of days to store rds snapshot on S3, defaults to 31." - default = 31 + description = "Number of days to store rds snapshot on S3, defaults to 14." + default = 14 } variable "enable_s3_load_balancer_logs_lifecycle" { @@ -511,8 +511,8 @@ variable "enable_s3_load_balancer_logs_lifecycle" { variable "s3_load_balancer_logs_expiration_days" { type = number - description = "Number of days to store load balancer logs on S3, defaults to 31." - default = 31 + description = "Number of days to store load balancer logs on S3, defaults to 14." + default = 14 } variable "create_read_replica_2" { From 39b0eb9a810329b0d478c0cd1b79094a05f95da9 Mon Sep 17 00:00:00 2001 From: owl-king Date: Fri, 9 Aug 2024 09:48:56 +0700 Subject: [PATCH 10/25] Change full_node_root_block_device_size default value to 1000 GB (#122) --- indexer/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/variables.tf b/indexer/variables.tf index cc4f6f25..56575de5 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -165,7 +165,7 @@ variable "full_node_container_chain_home" { variable "full_node_root_block_device_size" { type = number description = "Size of root block device in gigabytes" - default = 3000 + default = 1000 } variable "snapshot_full_node_container_chain_home" { From e69975b845007fe479a177c8a835d09d1b743879 Mon Sep 17 00:00:00 2001 From: owl-king Date: Fri, 9 Aug 2024 10:00:32 +0700 Subject: [PATCH 11/25] Add new variable: msk_storage_size to set storage of kafka (#123) --- indexer/variables.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/indexer/variables.tf b/indexer/variables.tf index 56575de5..59f0d1e4 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -105,6 +105,12 @@ variable "msk_instance_type" { description = "Instance type for MSK brokers" } +variable "msk_storage_size" { + type = string + description = "Storage size of MSK nodes. Suggested value: 2000 for mainnet, 1000 for staging and testnet and 500 for dev." + default = "500" +} + variable "rds_db_instance_class" { type = string description = "Instance class for the Postgres RDS DB" @@ -565,3 +571,4 @@ variable "lambda_cpu_architecture" { error_message = "Err: invalid environment. Must be one of {X86_64 | ARM64}." } } + From d6b364428bcf3ace6c61fd21201b168563fa7e03 Mon Sep 17 00:00:00 2001 From: owl-king Date: Fri, 9 Aug 2024 15:07:12 +0700 Subject: [PATCH 12/25] Allow to disable backup_full_node (#124) --- indexer/backup_full_node_ap_northeast_1.tf | 6 ++++++ indexer/route_table.tf | 9 +++++---- indexer/security_group.tf | 2 +- indexer/variables.tf | 5 +++++ indexer/vpc.tf | 3 ++- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/indexer/backup_full_node_ap_northeast_1.tf b/indexer/backup_full_node_ap_northeast_1.tf index 6a59bce7..d248edf8 100644 --- a/indexer/backup_full_node_ap_northeast_1.tf +++ b/indexer/backup_full_node_ap_northeast_1.tf @@ -1,5 +1,6 @@ module "backup_full_node_ap_northeast_1" { source = "../modules/validator" + count = var.create_backup_full_node ? 1 : 0 environment = var.environment @@ -45,3 +46,8 @@ module "backup_full_node_ap_northeast_1" { aws = aws.ap_northeast_1 } } + +moved { + from = module.backup_full_node_ap_northeast_1 + to = module.backup_full_node_ap_northeast_1[0] +} diff --git a/indexer/route_table.tf b/indexer/route_table.tf index b19491e2..ce37872d 100644 --- a/indexer/route_table.tf +++ b/indexer/route_table.tf @@ -68,9 +68,10 @@ resource "aws_route" "full_node_route_to_indexer" { # NOTE: This is not an individual AWS resource, but rather an attachment to the route table, and so # no tags are added. resource "aws_route" "backup_full_node_route_to_indexer" { - route_table_id = module.backup_full_node_ap_northeast_1.route_table_id + count = var.create_backup_full_node ? 1 : 0 + route_table_id = module.backup_full_node_ap_northeast_1[0].route_table_id destination_cidr_block = var.indexers[var.region].vpc_cidr_block - vpc_peering_connection_id = aws_vpc_peering_connection.backup_full_node_peer.id + vpc_peering_connection_id = aws_vpc_peering_connection.backup_full_node_peer[0].id } # Route from the Indexer's private subnets to the full node's VPC. Needed so that the full node can @@ -88,9 +89,9 @@ resource "aws_route" "indexer_route_to_full_node" { } resource "aws_route" "indexer_route_to_backup_full_node" { - for_each = aws_route_table.private + for_each = var.create_backup_full_node ? aws_route_table.private : {} route_table_id = each.value.id destination_cidr_block = var.backup_full_node_cidr_vpc - vpc_peering_connection_id = aws_vpc_peering_connection.backup_full_node_peer.id + vpc_peering_connection_id = aws_vpc_peering_connection.backup_full_node_peer[0].id } diff --git a/indexer/security_group.tf b/indexer/security_group.tf index 08d8cd59..c2e1a79c 100644 --- a/indexer/security_group.tf +++ b/indexer/security_group.tf @@ -62,7 +62,7 @@ resource "aws_security_group" "msk" { security_groups = flatten([ aws_security_group.devbox.id, module.full_node_ap_northeast_1.aws_security_group_id, - module.backup_full_node_ap_northeast_1.aws_security_group_id, + var.create_backup_full_node ? [module.backup_full_node_ap_northeast_1[0].aws_security_group_id] : [], # Lambda Services [ for service in keys(local.lambda_services) : diff --git a/indexer/variables.tf b/indexer/variables.tf index 59f0d1e4..f4ba3b73 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -572,3 +572,8 @@ variable "lambda_cpu_architecture" { } } +variable "create_backup_full_node" { + description = "Create backup full node. Default: true for mainnet, testnet" + type = bool + default = true +} diff --git a/indexer/vpc.tf b/indexer/vpc.tf index dbb30339..6ae30168 100644 --- a/indexer/vpc.tf +++ b/indexer/vpc.tf @@ -84,8 +84,9 @@ resource "aws_vpc_peering_connection" "full_node_peer" { } resource "aws_vpc_peering_connection" "backup_full_node_peer" { + count = var.create_backup_full_node ? 1 : 0 peer_vpc_id = aws_vpc.main.id - vpc_id = module.backup_full_node_ap_northeast_1.aws_vpc_id + vpc_id = module.backup_full_node_ap_northeast_1[0].aws_vpc_id # Auto-accept allows the VPC peering connection to be made programmatically with no manual steps # to accept the VPC peering connection in the console # This can only be done if both VPCs are in the same region and AWS account (which they are) From 18b88b5b66703c60ef232b66d4bb2e54079d27f0 Mon Sep 17 00:00:00 2001 From: owl-king Date: Fri, 9 Aug 2024 15:29:58 +0700 Subject: [PATCH 13/25] Fix image filter value to find image based on cpu architecture (#125) --- modules/validator/ec2.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/validator/ec2.tf b/modules/validator/ec2.tf index a4894320..cc3279eb 100644 --- a/modules/validator/ec2.tf +++ b/modules/validator/ec2.tf @@ -27,7 +27,7 @@ data "aws_ami" "amazon_linux_ecs_ami" { filter { name = "name" - values = ["amzn2-ami-ecs-inf-hvm-*-x86_64-ebs"] + values = [var.ecs_task_cpu_architecture == "X86_64" ? "amzn2-ami-ecs-inf-hvm-*-x86_64-ebs" : "amzn2-ami-ecs-hvm-*-arm64-ebs"] } } From 10ff2023c1c8918b628b3e458762bfffa14de156 Mon Sep 17 00:00:00 2001 From: owl-king Date: Mon, 12 Aug 2024 15:28:08 +0700 Subject: [PATCH 14/25] Change default value of create_backup_full_node to false (#127) --- indexer/variables.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/indexer/variables.tf b/indexer/variables.tf index f4ba3b73..f3a3ad84 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -573,7 +573,7 @@ variable "lambda_cpu_architecture" { } variable "create_backup_full_node" { - description = "Create backup full node. Default: true for mainnet, testnet" + description = "Create backup full node. Default: false for all envs test and dev environment. Mainnet and Testnet should enable it." type = bool - default = true + default = false } From 7e171e616a0d117d29a4c3efda116210a7f402a8 Mon Sep 17 00:00:00 2001 From: Vuong Date: Sun, 25 Aug 2024 16:10:12 +0700 Subject: [PATCH 15/25] Change msk storage to msk_storage_size --- indexer/msk.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/indexer/msk.tf b/indexer/msk.tf index c0aca667..3d74cadb 100644 --- a/indexer/msk.tf +++ b/indexer/msk.tf @@ -36,7 +36,7 @@ resource "aws_msk_cluster" "main" { instance_type = var.msk_instance_type storage_info { ebs_storage_info { - volume_size = var.environment == "mainnet" ? 2000 : (contains(["dev", "dev2", "dev3", "dev4", "dev5"], var.environment) ? 500 : 1000) # in GB + volume_size = var.msk_storage_size } } client_subnets = [ @@ -56,4 +56,5 @@ resource "aws_msk_cluster" "main" { arn = aws_msk_configuration.main.arn revision = aws_msk_configuration.main.latest_revision } -} \ No newline at end of file +} + From 81673990677c3d0a03cb9ed3c17ced3fab1f4777 Mon Sep 17 00:00:00 2001 From: Vuong Date: Wed, 28 Aug 2024 12:05:53 +0700 Subject: [PATCH 16/25] Change default storage of full_node_snapshot to 1000 --- indexer/lambda.tf | 2 +- indexer/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/indexer/lambda.tf b/indexer/lambda.tf index db71fa2a..42faf841 100644 --- a/indexer/lambda.tf +++ b/indexer/lambda.tf @@ -13,7 +13,7 @@ resource "aws_lambda_function" "main" { function_name = "${each.key}_lambda_function" role = aws_iam_role.lambda_services[each.key].arn architectures = [lower(var.lambda_cpu_architecture)] - timeout = 120 + timeout = 300 environment { variables = merge( diff --git a/indexer/variables.tf b/indexer/variables.tf index f3a3ad84..cffe8226 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -213,7 +213,7 @@ variable "full_node_snapshot_upload_period" { variable "full_node_snapshot_ebs_volume_size" { type = number description = "Size (in GiB) of the EBS volume used for the fast sync full node" - default = 3000 + default = 1000 } variable "full_node_ec2_instance_type" { From 13822e49c2358b593ab117b088314507e0d344bc Mon Sep 17 00:00:00 2001 From: roy-dydx <133032749+roy-dydx@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:11:21 -0400 Subject: [PATCH 17/25] Remove apne-1a az (#132) --- indexer/terraform.tfvars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/terraform.tfvars b/indexer/terraform.tfvars index 96ecf5d4..02ca0eaa 100644 --- a/indexer/terraform.tfvars +++ b/indexer/terraform.tfvars @@ -1,7 +1,7 @@ indexers = { ap-northeast-1 = { name = "indexer-apne1" - availability_zones = ["ap-northeast-1a", "ap-northeast-1c", "ap-northeast-1d"] + availability_zones = ["ap-northeast-1c", "ap-northeast-1d"] vpc_cidr_block = "10.0.0.0/16" private_subnets_availability_zone_to_cidr_block = { "ap-northeast-1a" = "10.0.0.0/24" From bd21af51181688cd0e1ad5024d3571a395fcfe14 Mon Sep 17 00:00:00 2001 From: roy-dydx <133032749+roy-dydx@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:57:19 -0400 Subject: [PATCH 18/25] Revert "Remove apne-1a az (#132)" (#133) This reverts commit 6c743321fe34dde9be7df84e827b0c2774d81705. --- indexer/terraform.tfvars | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/indexer/terraform.tfvars b/indexer/terraform.tfvars index 02ca0eaa..96ecf5d4 100644 --- a/indexer/terraform.tfvars +++ b/indexer/terraform.tfvars @@ -1,7 +1,7 @@ indexers = { ap-northeast-1 = { name = "indexer-apne1" - availability_zones = ["ap-northeast-1c", "ap-northeast-1d"] + availability_zones = ["ap-northeast-1a", "ap-northeast-1c", "ap-northeast-1d"] vpc_cidr_block = "10.0.0.0/16" private_subnets_availability_zone_to_cidr_block = { "ap-northeast-1a" = "10.0.0.0/24" From 30ef8a6e2a3433cee22afd56b3993b7cdc73176e Mon Sep 17 00:00:00 2001 From: jerryfan01234 <44346807+jerryfan01234@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:21:40 -0400 Subject: [PATCH 19/25] [OTE-821] Add roundtable monitors for update affiliate info and update wallet total volume (#134) * upgrade kafka version and reduce session timeout * Add partition level logging for mainnet MSK * Fix terraform apply resource conflict * Add partition offset plot to vulcan dashboard * dummy * Add stale compliance data monitor for mainnet * add roundtable monitors for update affiliate info and update wallet total volume --- indexer/msk.tf | 2 +- .../indexer_monitors/roundtable_monitors.tf | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 modules/indexer_monitors/roundtable_monitors.tf diff --git a/indexer/msk.tf b/indexer/msk.tf index 3d74cadb..b3d75f7f 100644 --- a/indexer/msk.tf +++ b/indexer/msk.tf @@ -19,6 +19,7 @@ resource "aws_msk_configuration" "main" { message.max.bytes=4194304 unclean.leader.election.enable=true zookeeper.session.timeout.ms=6000 + replica.selector.class = org.apache.kafka.common.replica.RackAwareReplicaSelector PROPERTIES lifecycle { @@ -57,4 +58,3 @@ resource "aws_msk_cluster" "main" { revision = aws_msk_configuration.main.latest_revision } } - diff --git a/modules/indexer_monitors/roundtable_monitors.tf b/modules/indexer_monitors/roundtable_monitors.tf new file mode 100644 index 00000000..b16972aa --- /dev/null +++ b/modules/indexer_monitors/roundtable_monitors.tf @@ -0,0 +1,51 @@ +resource "datadog_monitor_json" "roundtable_update_affiliate_info_persistent_cache_stale" { + monitor = < 600", + "message": "persistentCache.affiliateInfoUpdateTime is more than 10 minutes in the past. This indicates that update-affiliate-info roundtable has not run successfully in past 10 min -> affiliate_info table is stale.", + "tags": [ + "team:${var.team}", + "env:${var.env_tag}" + ], + "options": { + "thresholds": { + "critical": 600 + }, + "notify_audit": false, + "include_tags": false, + "notify_no_data": false, + "silenced": {} + }, + "priority": null, + "restricted_roles": null +} +EOF +} + +resource "datadog_monitor_json" "roundtable_update_wallet_total_volume_persistent_cache_stale" { + monitor = < 600", + "message": "persistentCache.totalVolumeUpdateTime is more than 10 minutes in the past. This indicates that update-wallet-total-volume roundtable has not run successfully in past 10 min -> totalVolume column of wallets table is stale.", + "tags": [ + "team:${var.team}", + "env:${var.env_tag}" + ], + "options": { + "thresholds": { + "critical": 600 + }, + "notify_audit": false, + "include_tags": false, + "notify_no_data": false, + "silenced": {} + }, + "priority": null, + "restricted_roles": null +} +EOF +} \ No newline at end of file From b2c4e325ab73b1c1de18afff7016b15b059a8ed1 Mon Sep 17 00:00:00 2001 From: roy-dydx <133032749+roy-dydx@users.noreply.github.com> Date: Fri, 27 Sep 2024 13:00:18 -0400 Subject: [PATCH 20/25] Add AWS_REGION envvar to services that connect to kafka (#136) --- indexer/locals.tf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/indexer/locals.tf b/indexer/locals.tf index d0e57fff..c071a08e 100644 --- a/indexer/locals.tf +++ b/indexer/locals.tf @@ -38,6 +38,10 @@ locals { should_deploy_in_rds_subnet : true, ecs_environment_variables : flatten( [ + { + name : "AWS_REGION", + value : var.region, + }, { name : "PG_POOL_MAX", value : "30" @@ -109,6 +113,10 @@ locals { should_deploy_in_rds_subnet : false, ecs_environment_variables : flatten( [ + { + name : "AWS_REGION", + value : var.region, + }, { name : "COMLINK_URL", value : aws_lb.public.dns_name, @@ -207,6 +215,10 @@ locals { should_deploy_in_rds_subnet : false, ecs_environment_variables : flatten( [ + { + name : "AWS_REGION", + value : var.region, + }, { name : "PG_POOL_MAX", value : "2" From 29a24e2599846412af7b791aa2f86dca9eaa83f7 Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:25:25 -0400 Subject: [PATCH 21/25] Add read replica storage variable (#137) * add read replica storage var * set default --- indexer/locals.tf | 6 +++--- indexer/rds.tf | 2 ++ indexer/variables.tf | 24 ++++++++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/indexer/locals.tf b/indexer/locals.tf index c071a08e..e398b6b9 100644 --- a/indexer/locals.tf +++ b/indexer/locals.tf @@ -55,7 +55,7 @@ locals { ), }, "${local.service_names["comlink"]}" : { - ecs_desired_count : 5, + ecs_desired_count : var.comlink_ecs_desired_count, task_definition_memory : 4096, task_definition_cpu : 2048, is_public_facing : true, @@ -101,7 +101,7 @@ locals { ), }, "${local.service_names["socks"]}" : { - ecs_desired_count : 5, + ecs_desired_count : var.socks_ecs_desired_count, task_definition_memory : 8192, task_definition_cpu : 4096, is_public_facing : true, @@ -203,7 +203,7 @@ locals { ), }, "${local.service_names["vulcan"]}" : { - ecs_desired_count : 5, + ecs_desired_count : var.vulcan_ecs_desired_count, task_definition_memory : 8192, task_definition_cpu : 4096, is_public_facing : false, diff --git a/indexer/rds.tf b/indexer/rds.tf index bfa4316e..ee6771cf 100644 --- a/indexer/rds.tf +++ b/indexer/rds.tf @@ -231,6 +231,7 @@ resource "aws_db_instance" "read_replica" { # be specified for a replica, and will match the properties on the source db. vpc_security_group_ids = [aws_security_group.rds.id] parameter_group_name = aws_db_parameter_group.main.name + allocated_storage = var.rds_read_replica_db_allocated_storage_gb publicly_accessible = false # Set to true if any planned changes need to be applied before the next maintenance window. apply_immediately = false @@ -257,6 +258,7 @@ resource "aws_db_instance" "read_replica_2" { # be specified for a replica, and will match the properties on the source db. vpc_security_group_ids = [aws_security_group.rds.id] parameter_group_name = aws_db_parameter_group.main.name + allocated_storage = var.rds_read_replica_db_allocated_storage_gb publicly_accessible = false # Set to true if any planned changes need to be applied before the next maintenance window. apply_immediately = false diff --git a/indexer/variables.tf b/indexer/variables.tf index cffe8226..d822628c 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -121,6 +121,12 @@ variable "rds_db_allocated_storage_gb" { description = "Storage allocated to the Postgres RDS DB in GB" } +variable "rds_read_replica_db_allocated_storage_gb" { + type = number + description = "Storage allocated to the Postgres RDS read replica DBs in GB" + default = 1000 +} + variable "elasticache_redis_num_cache_clusters" { type = number description = "Number of elasticache cache clusters" @@ -577,3 +583,21 @@ variable "create_backup_full_node" { type = bool default = false } + +variable "vulcan_ecs_desired_count" { + type = number + description = "Number of desired vulcan instances." + default = 5 +} + +variable "comlink_ecs_desired_count" { + type = number + description = "Number of desired comlinnk instances." + default = 5 +} + +variable "socks_ecs_desired_count" { + type = number + description = "Number of desired socks instances." + default = 5 +} From 6e058af6d3f0d2951fe8f475aca692e8a139f798 Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:43:05 -0400 Subject: [PATCH 22/25] Remove Indexer dashboards from terraform (#139) --- indexer_dashboards/indexer_dashboards.tf | 4 - indexer_dashboards/providers.tf | 6 - indexer_dashboards/terraform.tfvars | 51 - indexer_dashboards/variables.tf | 40 - indexer_dashboards/versions.tf | 18 - .../indexer_dashboards/comlink_dashboard.tf | 822 ---- modules/indexer_dashboards/ender_dashboard.tf | 3354 ----------------- .../indexer_dashboards/full_node_dashboard.tf | 544 --- modules/indexer_dashboards/locals.tf | 69 - .../indexer_dashboards/postgres_dashboard.tf | 66 - .../roundtable_dashboard.tf | 952 ----- modules/indexer_dashboards/socks_dashboard.tf | 1526 -------- modules/indexer_dashboards/variables.tf | 22 - modules/indexer_dashboards/versions.tf | 10 - .../indexer_dashboards/vulcan_dashboard.tf | 1813 --------- 15 files changed, 9297 deletions(-) delete mode 100644 indexer_dashboards/indexer_dashboards.tf delete mode 100644 indexer_dashboards/providers.tf delete mode 100644 indexer_dashboards/terraform.tfvars delete mode 100644 indexer_dashboards/variables.tf delete mode 100644 indexer_dashboards/versions.tf delete mode 100644 modules/indexer_dashboards/comlink_dashboard.tf delete mode 100644 modules/indexer_dashboards/ender_dashboard.tf delete mode 100644 modules/indexer_dashboards/full_node_dashboard.tf delete mode 100644 modules/indexer_dashboards/locals.tf delete mode 100644 modules/indexer_dashboards/postgres_dashboard.tf delete mode 100644 modules/indexer_dashboards/roundtable_dashboard.tf delete mode 100644 modules/indexer_dashboards/socks_dashboard.tf delete mode 100644 modules/indexer_dashboards/variables.tf delete mode 100644 modules/indexer_dashboards/versions.tf delete mode 100644 modules/indexer_dashboards/vulcan_dashboard.tf diff --git a/indexer_dashboards/indexer_dashboards.tf b/indexer_dashboards/indexer_dashboards.tf deleted file mode 100644 index 031fed71..00000000 --- a/indexer_dashboards/indexer_dashboards.tf +++ /dev/null @@ -1,4 +0,0 @@ -module "indexer_dashboards" { - source = "../modules/indexer_dashboards" - indexer_services_variable_mapping = var.indexer_services_variable_mapping -} diff --git a/indexer_dashboards/providers.tf b/indexer_dashboards/providers.tf deleted file mode 100644 index 8d83a778..00000000 --- a/indexer_dashboards/providers.tf +++ /dev/null @@ -1,6 +0,0 @@ -# Default provider. -provider "datadog" { - api_key = var.datadog_api_key - app_key = var.datadog_app_key - api_url = var.datadog_api_url -} diff --git a/indexer_dashboards/terraform.tfvars b/indexer_dashboards/terraform.tfvars deleted file mode 100644 index d83fce12..00000000 --- a/indexer_dashboards/terraform.tfvars +++ /dev/null @@ -1,51 +0,0 @@ -indexer_services_variable_mapping = { - "dev" : { - "environment" : "dev", - "service" : "indexer", - "cluster_name" : "dev-indexer-apne1-cluster", - "ecs_cluster_name" : "dev-indexer-full-node-cluster", - "msk_cluster_name" : "dev-indexer-apne1-msk-cluster" - }, - "dev2" : { - "environment" : "dev2", - "service" : "indexer", - "cluster_name" : "dev2-indexer-apne1-cluster", - "ecs_cluster_name" : "dev2-indexer-full-node-cluster", - "msk_cluster_name" : "dev2-indexer-apne1-msk-cluster" - }, - "dev3" : { - "environment" : "dev3", - "service" : "indexer", - "cluster_name" : "dev3-indexer-apne1-cluster", - "ecs_cluster_name" : "dev3-indexer-full-node-cluster", - "msk_cluster_name" : "dev3-indexer-apne1-msk-cluster" - }, - "dev4" : { - "environment" : "dev4", - "service" : "indexer", - "cluster_name" : "dev4-indexer-apne1-cluster", - "ecs_cluster_name" : "dev4-indexer-full-node-cluster", - "msk_cluster_name" : "dev4-indexer-apne1-msk-cluster" - }, - "dev5" : { - "environment" : "dev5", - "service" : "indexer", - "cluster_name" : "dev5-indexer-apne1-cluster", - "ecs_cluster_name" : "dev5-indexer-full-node-cluster", - "msk_cluster_name" : "dev5-indexer-apne1-msk-cluster" - }, - "staging" : { - "environment" : "staging", - "service" : "indexer", - "cluster_name" : "staging-indexer-apne1-cluster", - "ecs_cluster_name" : "staging-indexer-full-node-cluster", - "msk_cluster_name" : "staging-indexer-apne1-msk-cluster" - }, - "testnet2" : { - "environment" : "testnet2", - "service" : "indexer", - "cluster_name" : "testnet2-indexer-apne1-cluster", - "ecs_cluster_name" : "testnet2-indexer-full-node-cluster", - "msk_cluster_name" : "testnet2-indexer-apne1-msk-cluster" - }, -} diff --git a/indexer_dashboards/variables.tf b/indexer_dashboards/variables.tf deleted file mode 100644 index 1f9bd1fd..00000000 --- a/indexer_dashboards/variables.tf +++ /dev/null @@ -1,40 +0,0 @@ -variable "indexer_services_variable_mapping" { - type = map( - object({ - # Environment - environment = string - - # Service name - service = string - - # Cluster name for the indexer services - cluster_name = string - - # ECS cluster name for the full node - ecs_cluster_name = string - - # MSK cluster name - msk_cluster_name = string - }) - ) - - description = "Map of variable name to preset values of variables used in indexer services." -} - -variable "datadog_api_key" { - type = string - description = "Datadog API key" - sensitive = true -} - -variable "datadog_app_key" { - type = string - description = "Datadog app key" - sensitive = true -} - -variable "datadog_api_url" { - type = string - description = "The datadog api url" - default = "https://api.datadoghq.com/" -} diff --git a/indexer_dashboards/versions.tf b/indexer_dashboards/versions.tf deleted file mode 100644 index 401824f8..00000000 --- a/indexer_dashboards/versions.tf +++ /dev/null @@ -1,18 +0,0 @@ -terraform { - cloud { - organization = "dydxprotocol" - - workspaces { - tags = ["indexer-dashboards"] - } - } - - required_providers { - datadog = { - source = "DataDog/datadog" - version = "~> 3.29" - } - } - - required_version = "~> 1.3.2" -} diff --git a/modules/indexer_dashboards/comlink_dashboard.tf b/modules/indexer_dashboards/comlink_dashboard.tf deleted file mode 100644 index c3c7b154..00000000 --- a/modules/indexer_dashboards/comlink_dashboard.tf +++ /dev/null @@ -1,822 +0,0 @@ -resource "datadog_dashboard_json" "comlink" { - dashboard = < Date: Mon, 30 Sep 2024 16:59:29 -0400 Subject: [PATCH 23/25] Add variable for full_node root_block_device_size (#141) * Update full_node_ap_northeast_1.tf * Update variables.tf From 105efcef042812b2f5d46a39a661ade398276eb6 Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Wed, 2 Oct 2024 14:48:01 -0400 Subject: [PATCH 24/25] set log retention hours to 120 (#142) --- indexer/msk.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/indexer/msk.tf b/indexer/msk.tf index b3d75f7f..55c05d95 100644 --- a/indexer/msk.tf +++ b/indexer/msk.tf @@ -20,6 +20,7 @@ resource "aws_msk_configuration" "main" { unclean.leader.election.enable=true zookeeper.session.timeout.ms=6000 replica.selector.class = org.apache.kafka.common.replica.RackAwareReplicaSelector + log.retention.hours = 120 PROPERTIES lifecycle { From 239a52aec5c4697dcb6c4277af07f984cffcd3ab Mon Sep 17 00:00:00 2001 From: dydxwill <119354122+dydxwill@users.noreply.github.com> Date: Thu, 3 Oct 2024 14:08:07 -0400 Subject: [PATCH 25/25] add default (#143) --- indexer/variables.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/indexer/variables.tf b/indexer/variables.tf index d822628c..b8981d62 100644 --- a/indexer/variables.tf +++ b/indexer/variables.tf @@ -119,6 +119,7 @@ variable "rds_db_instance_class" { variable "rds_db_allocated_storage_gb" { type = number description = "Storage allocated to the Postgres RDS DB in GB" + default = 1000 } variable "rds_read_replica_db_allocated_storage_gb" {