feat(k8s): run adhoc jobs in dedicated namespace (#1065)

* feat(k8s): run adhoc jobs in dedicated namespace * feat: add ttl for all jobs
wmde · Aug 14, 2023 · 7c2995d · 7c2995d
1 parent e038ba8
commit 7c2995d
Show file tree

Hide file tree

Showing 16 changed files with 98 additions and 9 deletions.
diff --git a/k8s/jobs/changeReplicationPasswordOnSecondary.yaml b/k8s/jobs/changeReplicationPasswordOnSecondary.yaml
@@ -2,8 +2,10 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   generateName: change-replication-password-on-secondary-
+  namespace: adhoc-jobs
 spec:
   template:
+    ttlSecondsAfterFinished: 604800
     spec:
       ttlSecondsAfterFinished: 604800
       containers:

diff --git a/k8s/jobs/elasticSearchImportJob.yaml b/k8s/jobs/elasticSearchImportJob.yaml
@@ -2,8 +2,10 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   generateName: load-elasticsearch-data-
+  namespace: adhoc-jobs
 spec:
   template:
+    ttlSecondsAfterFinished: 604800
     metadata:
       name: load-elasticsearch-data
     spec:

diff --git a/k8s/jobs/forceSearchIndexFrom.yaml b/k8s/jobs/forceSearchIndexFrom.yaml
@@ -2,8 +2,10 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   generateName: force-search-index-from-
+  namespace: adhoc-jobs
 spec:
   template:
+    ttlSecondsAfterFinished: 604800
     metadata:
       name: force-search-index-from
     spec:

diff --git a/k8s/jobs/rebuildQuantityUnitsJob.yaml b/k8s/jobs/rebuildQuantityUnitsJob.yaml
@@ -2,8 +2,10 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   generateName: rebuild-quantity-units-
+  namespace: adhoc-jobs
 spec:
   template:
+    ttlSecondsAfterFinished: 604800
     metadata:
       name: rebuild-quantity-units
     spec:

diff --git a/k8s/jobs/resetOtherSqlSecretsJob.yaml b/k8s/jobs/resetOtherSqlSecretsJob.yaml
@@ -2,8 +2,10 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   generateName: reset-other-sql-secrets-job-
+  namespace: adhoc-jobs
 spec:
   template:
+    ttlSecondsAfterFinished: 604800
     spec:
       ttlSecondsAfterFinished: 604800
       containers:

diff --git a/k8s/jobs/resetRootSqlSecretJob.yaml b/k8s/jobs/resetRootSqlSecretJob.yaml
@@ -2,8 +2,10 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   generateName: reset-root-sql-secret-job-
+  namespace: adhoc-jobs
 spec:
   template:
+    ttlSecondsAfterFinished: 604800
     spec:
       ttlSecondsAfterFinished: 604800
       containers:

diff --git a/k8s/jobs/runAllMWJobsJob.yaml b/k8s/jobs/runAllMWJobsJob.yaml
@@ -2,8 +2,10 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   generateName: run-all-mw-jobs-
+  namespace: adhoc-jobs
 spec:
   template:
+    ttlSecondsAfterFinished: 604800
     metadata:
       name: run-all-mw-jobs
     spec:

diff --git a/tf/env/local/namespaces.tf b/tf/env/local/namespaces.tf
@@ -16,3 +16,23 @@ resource "kubernetes_resource_quota" "api-jobs-podquota" {
     scopes = ["BestEffort"]
   }
 }
+
+
+resource "kubernetes_namespace" "adhoc-job-namespace" {
+  metadata {
+    name = "adhoc-jobs"
+  }
+}
+
+resource "kubernetes_resource_quota" "adhoc-jobs-podquota" {
+  metadata {
+    name      = "api-jobs-podquota"
+    namespace = kubernetes_namespace.api-job-namespace.metadata[0].name
+  }
+  spec {
+    hard = {
+      pods = 1
+    }
+    scopes = ["BestEffort"]
+  }
+}
diff --git a/tf/env/local/secrets-api.tf b/tf/env/local/secrets-api.tf
@@ -29,7 +29,7 @@ resource "random_password" "api-app-jwt-secret" {
 }
 
 resource "kubernetes_secret" "api-app-secrets" {
-  for_each = toset(["default", "api-jobs"])
+  for_each = toset(["default", "api-jobs", "adhoc-jobs"])
   metadata {
     name      = "api-app-secrets"
     namespace = each.value

diff --git a/tf/env/local/secrets-recapcha.tf b/tf/env/local/secrets-recapcha.tf
@@ -1,5 +1,5 @@
 resource "kubernetes_secret" "recaptcha-v3-dev-secrets" {
-  for_each = toset(["default", "api-jobs"])
+  for_each = toset(["default", "api-jobs", "adhoc-jobs"])
   metadata {
     name = "recaptcha-v3-dev-secrets"
     # default as staging
@@ -18,7 +18,7 @@ moved {
 }
 
 resource "kubernetes_secret" "recaptcha-v2-dev-secrets" {
-  for_each = toset(["default", "api-jobs"])
+  for_each = toset(["default", "api-jobs", "adhoc-jobs"])
   metadata {
     name = "recaptcha-v2-dev-secrets"
     # default as staging

diff --git a/tf/env/local/secrets-redis.tf b/tf/env/local/secrets-redis.tf
@@ -7,7 +7,7 @@ resource "random_password" "redis-password" {
 
 # Used by the sql service for initial setup
 resource "kubernetes_secret" "redis-password" {
-  for_each = toset(["default", "api-jobs"])
+  for_each = toset(["default", "api-jobs", "adhoc-jobs"])
   metadata {
     name      = "redis-password"
     namespace = each.value

diff --git a/tf/env/local/secrets-sql.tf b/tf/env/local/secrets-sql.tf
@@ -8,7 +8,7 @@ resource "random_password" "sql-passwords" {
 
 # Used by the sql service for initial setup
 resource "kubernetes_secret" "sql-secrets-passwords" {
-  for_each = toset(["default", "api-jobs"])
+  for_each = toset(["default", "api-jobs", "adhoc-jobs"])
   metadata {
     name      = "sql-secrets-passwords"
     namespace = each.value
@@ -27,7 +27,7 @@ moved {
 
 # Used by the init script on sql services for user and permissions setup
 resource "kubernetes_secret" "sql-secrets-init-passwords" {
-  for_each = toset(["default", "api-jobs"])
+  for_each = toset(["default", "api-jobs", "adhoc-jobs"])
   metadata {
     name      = "sql-secrets-init-passwords"
     namespace = each.value

diff --git a/tf/env/production/kubernetes-secrets.tf b/tf/env/production/kubernetes-secrets.tf
@@ -21,7 +21,11 @@ module "wbaas-k8s-secrets" {
   api_passport_private_key          = tls_private_key.api-passport.private_key_pem
   api_app_key                       = random_password.api-app-key.result
   api_app_jwt_secret                = random_password.api-app-jwt-secret.result
-  mediawiki_secret_namespaces       = ["default", kubernetes_namespace.api-job-namespace.metadata[0].name]
+  mediawiki_secret_namespaces = [
+    "default",
+    kubernetes_namespace.api-job-namespace.metadata[0].name,
+    kubernetes_namespace.adhoc-job-namespace.metadata[0].name
+  ]
   logical_backup_openssl_secret     = random_password.logical_backup_random_password.result
 }
 
diff --git a/tf/env/production/namespaces.tf b/tf/env/production/namespaces.tf
@@ -20,3 +20,27 @@ resource "kubernetes_resource_quota" "api-jobs-podquota" {
     scopes = ["BestEffort"]
   }
 }
+
+
+resource "kubernetes_namespace" "adhoc-job-namespace" {
+  provider = kubernetes.wbaas-2
+
+  metadata {
+    name = "adhoc-jobs"
+  }
+}
+
+resource "kubernetes_resource_quota" "adhoc-jobs-podquota" {
+  provider = kubernetes.wbaas-2
+
+  metadata {
+    name      = "api-jobs-podquota"
+    namespace = kubernetes_namespace.api-job-namespace.metadata[0].name
+  }
+  spec {
+    hard = {
+      pods = 8
+    }
+    scopes = ["BestEffort"]
+  }
+}
diff --git a/tf/env/staging/kubernetes-secrets.tf b/tf/env/staging/kubernetes-secrets.tf
@@ -21,6 +21,10 @@ module "wbaas2-k8s-secrets" {
   api_passport_private_key          = tls_private_key.api-passport.private_key_pem
   api_app_key                       = random_password.api-app-key.result
   api_app_jwt_secret                = random_password.api-app-jwt-secret.result
-  mediawiki_secret_namespaces       = ["default", kubernetes_namespace.api-job-namespace.metadata[0].name]
-  logical_backup_openssl_secret     = random_password.logical_backup_random_password.result
+  mediawiki_secret_namespaces = [
+  "default",
+    kubernetes_namespace.api-job-namespace.metadata[0].name,
+  kubernetes_namespace.adhoc-job-namespace.metadata[0].name
+  ]
+  logical_backup_openssl_secret = random_password.logical_backup_random_password.result
 }
diff --git a/tf/env/staging/namespaces.tf b/tf/env/staging/namespaces.tf
@@ -20,3 +20,26 @@ resource "kubernetes_resource_quota" "api-jobs-podquota" {
     scopes = ["BestEffort"]
   }
 }
+
+resource "kubernetes_namespace" "adhoc-job-namespace" {
+  provider = kubernetes.wbaas-2
+
+  metadata {
+    name = "adhoc-jobs"
+  }
+}
+
+resource "kubernetes_resource_quota" "adhoc-jobs-podquota" {
+  provider = kubernetes.wbaas-2
+
+  metadata {
+    name      = "api-jobs-podquota"
+    namespace = kubernetes_namespace.api-job-namespace.metadata[0].name
+  }
+  spec {
+    hard = {
+      pods = 4
+    }
+    scopes = ["BestEffort"]
+  }
+}