From 3dfe4c66a60c40719a0048ffd9e2ae05e20d4b72 Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 11:10:02 +0200 Subject: [PATCH 01/13] Introduces podAntiAffinity to avoid co-location of the replicas on the same node --- statefulset.tf | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/statefulset.tf b/statefulset.tf index dce81db..dc23258 100755 --- a/statefulset.tf +++ b/statefulset.tf @@ -33,6 +33,23 @@ resource "kubernetes_stateful_set" "axonserver" { spec { termination_grace_period_seconds = 120 + affinity { + pod_anti_affinity { + preferred_during_scheduling_ignored_during_execution { + topology_key = "kubernetes.io/hostname" + pod_affinity_term { + label_selector { + match_expressions { + key = "app" + operator = "In" + values = ["${var.cluster_name}-${count.index + 1}"] + } + } + } + } + } + } + container { name = "${var.cluster_name}-${count.index + 1}" image = "axoniq/axonserver:${var.axonserver_release}-jdk-${var.java_version}" From 35874045ebb8f95c2efe84cfe333c7491cd2d513 Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 11:13:29 +0200 Subject: [PATCH 02/13] Moved topology_key under pod_affinity_term --- statefulset.tf | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/statefulset.tf b/statefulset.tf index dc23258..45e0f99 100755 --- a/statefulset.tf +++ b/statefulset.tf @@ -36,8 +36,11 @@ resource "kubernetes_stateful_set" "axonserver" { affinity { pod_anti_affinity { preferred_during_scheduling_ignored_during_execution { - topology_key = "kubernetes.io/hostname" + weight = "20" + pod_affinity_term { + topology_key = "kubernetes.io/hostname" + label_selector { match_expressions { key = "app" From 6cc0b2d52d0f58f76a0b0ade7bd55aef857ef65a Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 11:48:08 +0200 Subject: [PATCH 03/13] Changed label_selector to cluster_name --- statefulset.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/statefulset.tf b/statefulset.tf index 45e0f99..584064c 100755 --- a/statefulset.tf +++ b/statefulset.tf @@ -43,9 +43,9 @@ resource "kubernetes_stateful_set" "axonserver" { label_selector { match_expressions { - key = "app" + key = "cluster" operator = "In" - values = ["${var.cluster_name}-${count.index + 1}"] + values = [ var.cluster_name ] } } } From 072d5c49d23c93e366fc2f9e32825fdf7fa3fd27 Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 12:01:00 +0200 Subject: [PATCH 04/13] Introduced avoid_colocation bool var --- statefulset.tf | 3 ++- variables.tf | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/statefulset.tf b/statefulset.tf index 584064c..651ae99 100755 --- a/statefulset.tf +++ b/statefulset.tf @@ -33,7 +33,8 @@ resource "kubernetes_stateful_set" "axonserver" { spec { termination_grace_period_seconds = 120 - affinity { + dynamic "affinity" { + for_each = var.avoid_colocation == "true" ? 1: 0 pod_anti_affinity { preferred_during_scheduling_ignored_during_execution { weight = "20" diff --git a/variables.tf b/variables.tf index adcdb25..7fb61d9 100755 --- a/variables.tf +++ b/variables.tf @@ -105,3 +105,9 @@ variable "devmode_enabled" { type = bool default = false } + +variable "avoid_colocation" { + description = "Avoid co location of the replicas on the same node" + type = bool + default = false +} From ecd689cb8d856d9b503d430ee1bee274bd7125bd Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 12:03:52 +0200 Subject: [PATCH 05/13] Forgot the content of the dynamic block --- statefulset.tf | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/statefulset.tf b/statefulset.tf index 651ae99..1986cfc 100755 --- a/statefulset.tf +++ b/statefulset.tf @@ -35,18 +35,20 @@ resource "kubernetes_stateful_set" "axonserver" { dynamic "affinity" { for_each = var.avoid_colocation == "true" ? 1: 0 - pod_anti_affinity { - preferred_during_scheduling_ignored_during_execution { - weight = "20" - - pod_affinity_term { - topology_key = "kubernetes.io/hostname" - - label_selector { - match_expressions { - key = "cluster" - operator = "In" - values = [ var.cluster_name ] + content { + pod_anti_affinity { + preferred_during_scheduling_ignored_during_execution { + weight = "20" + + pod_affinity_term { + topology_key = "kubernetes.io/hostname" + + label_selector { + match_expressions { + key = "cluster" + operator = "In" + values = [ var.cluster_name ] + } } } } From 9d85c38d72ae16ccb538198f6109a76848325fb4 Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 12:06:01 +0200 Subject: [PATCH 06/13] Misuse of avoid_colocation var --- statefulset.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statefulset.tf b/statefulset.tf index 1986cfc..12c45ce 100755 --- a/statefulset.tf +++ b/statefulset.tf @@ -34,7 +34,7 @@ resource "kubernetes_stateful_set" "axonserver" { termination_grace_period_seconds = 120 dynamic "affinity" { - for_each = var.avoid_colocation == "true" ? 1: 0 + for_each = var.avoid_colocation ? 1 : 0 content { pod_anti_affinity { preferred_during_scheduling_ignored_during_execution { From 2be3a7a58c63590cede5f9e2d0bd98feb7b36ba6 Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 12:18:18 +0200 Subject: [PATCH 07/13] Misuse of avoid_colocation var --- statefulset.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statefulset.tf b/statefulset.tf index 12c45ce..52ef304 100755 --- a/statefulset.tf +++ b/statefulset.tf @@ -34,7 +34,7 @@ resource "kubernetes_stateful_set" "axonserver" { termination_grace_period_seconds = 120 dynamic "affinity" { - for_each = var.avoid_colocation ? 1 : 0 + for_each = var.avoid_colocation ? {} : {} content { pod_anti_affinity { preferred_during_scheduling_ignored_during_execution { From 45329726a1b7a8335f706974a49280f4fb942be9 Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 12:21:07 +0200 Subject: [PATCH 08/13] Misuse of avoid_colocation var --- statefulset.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/statefulset.tf b/statefulset.tf index 52ef304..6172ce7 100755 --- a/statefulset.tf +++ b/statefulset.tf @@ -34,7 +34,7 @@ resource "kubernetes_stateful_set" "axonserver" { termination_grace_period_seconds = 120 dynamic "affinity" { - for_each = var.avoid_colocation ? {} : {} + for_each = var.avoid_colocation == true ? [1] : [ ] content { pod_anti_affinity { preferred_during_scheduling_ignored_during_execution { From 9f7eacf7e89fd288bae9268e38c3dea24669b84b Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 14:24:34 +0200 Subject: [PATCH 09/13] Refactoring of the co location var name --- statefulset.tf | 2 +- variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/statefulset.tf b/statefulset.tf index 6172ce7..ba70664 100755 --- a/statefulset.tf +++ b/statefulset.tf @@ -34,7 +34,7 @@ resource "kubernetes_stateful_set" "axonserver" { termination_grace_period_seconds = 120 dynamic "affinity" { - for_each = var.avoid_colocation == true ? [1] : [ ] + for_each = var.instance_co_location == true ? [ ] : [1] content { pod_anti_affinity { preferred_during_scheduling_ignored_during_execution { diff --git a/variables.tf b/variables.tf index 7fb61d9..62cc9ca 100755 --- a/variables.tf +++ b/variables.tf @@ -106,7 +106,7 @@ variable "devmode_enabled" { default = false } -variable "avoid_colocation" { +variable "instance_co_location" { description = "Avoid co location of the replicas on the same node" type = bool default = false From fb7b257cae60888d8228b7ba46e9d867d26b020e Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 14:27:07 +0200 Subject: [PATCH 10/13] Refactoring of the co location var name --- statefulset.tf | 2 +- variables.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/statefulset.tf b/statefulset.tf index ba70664..7534d8d 100755 --- a/statefulset.tf +++ b/statefulset.tf @@ -34,7 +34,7 @@ resource "kubernetes_stateful_set" "axonserver" { termination_grace_period_seconds = 120 dynamic "affinity" { - for_each = var.instance_co_location == true ? [ ] : [1] + for_each = var.share_instance == true ? [ ] : [1] content { pod_anti_affinity { preferred_during_scheduling_ignored_during_execution { diff --git a/variables.tf b/variables.tf index 62cc9ca..662689a 100755 --- a/variables.tf +++ b/variables.tf @@ -106,8 +106,8 @@ variable "devmode_enabled" { default = false } -variable "instance_co_location" { +variable "share_instance" { description = "Avoid co location of the replicas on the same node" type = bool - default = false + default = true } From 0241f2fa3f84b3d058c5882f27f1e6a155802531 Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 14:31:13 +0200 Subject: [PATCH 11/13] Refactoring of the co location var name --- statefulset.tf | 2 +- variables.tf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/statefulset.tf b/statefulset.tf index 7534d8d..c7ead5c 100755 --- a/statefulset.tf +++ b/statefulset.tf @@ -34,7 +34,7 @@ resource "kubernetes_stateful_set" "axonserver" { termination_grace_period_seconds = 120 dynamic "affinity" { - for_each = var.share_instance == true ? [ ] : [1] + for_each = var.assign_pods_to_different_nodes == true ? [1] : [ ] content { pod_anti_affinity { preferred_during_scheduling_ignored_during_execution { diff --git a/variables.tf b/variables.tf index 662689a..a5f324e 100755 --- a/variables.tf +++ b/variables.tf @@ -106,8 +106,8 @@ variable "devmode_enabled" { default = false } -variable "share_instance" { +variable "assign_pods_to_different_nodes" { description = "Avoid co location of the replicas on the same node" type = bool - default = true + default = false } From 0eb54037abf32fb13217d190d613c0d323f3a334 Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 14:34:41 +0200 Subject: [PATCH 12/13] Added the new var to the README.md --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a5caf74..e8c0747 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,17 @@ ```terraform module "as_demo" { - source = "git@github.com:AxonIQ/terraform-axonserver-k8s.git?ref=v1.9" + source = "git@github.com:AxonIQ/terraform-axonserver-k8s.git?ref=v1.10" - axonserver_release = "2024.1.0" + axonserver_release = "2024.1.1" java_version = "17" nodes_number = 3 cluster_name = "axonserver" public_domain = "axoniq.net" namespace = "axonserver" + + assign_pods_to_different_nodes = true axonserver_license_path = file("${path.module}/axoniq.license") axonserver_properties = file("${path.module}/axonserver.properties") @@ -57,6 +59,7 @@ module "as_demo" { | [data\_storage](#input\_data\_storage) | Data PVC storage | `string` | `"10Gi"` | no | | [plugins\_storage](#input\_plugins\_storage) | Plugins PVC storage | `string` | `"1Gi"` | no | | [devmode\_enabled](#input\_devmode\_enabled) | Axon Server devmode | `bool` | `false` | no | +| [assign\_pods\_to\_different\_nodes](#input\_assign\_pods\_to\_different\_nodes) | Avoid the co location of the replicas on the same node | `bool` | `false` | no | ## Outputs From 442fdd08bb4e3b6af9dc23bac67ab182b94e5da6 Mon Sep 17 00:00:00 2001 From: Francesco Latini Date: Wed, 28 Aug 2024 14:36:04 +0200 Subject: [PATCH 13/13] Removed assign_pods_to_different_nodes from the usage example --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index e8c0747..b2a7217 100644 --- a/README.md +++ b/README.md @@ -29,8 +29,6 @@ module "as_demo" { public_domain = "axoniq.net" namespace = "axonserver" - assign_pods_to_different_nodes = true - axonserver_license_path = file("${path.module}/axoniq.license") axonserver_properties = file("${path.module}/axonserver.properties") }