From 7c2995d014564c53718402fa42ff631b4638c45a Mon Sep 17 00:00:00 2001 From: Frederik Ring Date: Mon, 14 Aug 2023 15:58:44 +0200 Subject: [PATCH] feat(k8s): run adhoc jobs in dedicated namespace (#1065) * feat(k8s): run adhoc jobs in dedicated namespace * feat: add ttl for all jobs --- .../changeReplicationPasswordOnSecondary.yaml | 2 ++ k8s/jobs/elasticSearchImportJob.yaml | 2 ++ k8s/jobs/forceSearchIndexFrom.yaml | 2 ++ k8s/jobs/rebuildQuantityUnitsJob.yaml | 2 ++ k8s/jobs/resetOtherSqlSecretsJob.yaml | 2 ++ k8s/jobs/resetRootSqlSecretJob.yaml | 2 ++ k8s/jobs/runAllMWJobsJob.yaml | 2 ++ tf/env/local/namespaces.tf | 20 ++++++++++++++++ tf/env/local/secrets-api.tf | 2 +- tf/env/local/secrets-recapcha.tf | 4 ++-- tf/env/local/secrets-redis.tf | 2 +- tf/env/local/secrets-sql.tf | 4 ++-- tf/env/production/kubernetes-secrets.tf | 6 ++++- tf/env/production/namespaces.tf | 24 +++++++++++++++++++ tf/env/staging/kubernetes-secrets.tf | 8 +++++-- tf/env/staging/namespaces.tf | 23 ++++++++++++++++++ 16 files changed, 98 insertions(+), 9 deletions(-) diff --git a/k8s/jobs/changeReplicationPasswordOnSecondary.yaml b/k8s/jobs/changeReplicationPasswordOnSecondary.yaml index a14d4205d..aebf56084 100644 --- a/k8s/jobs/changeReplicationPasswordOnSecondary.yaml +++ b/k8s/jobs/changeReplicationPasswordOnSecondary.yaml @@ -2,8 +2,10 @@ apiVersion: batch/v1 kind: Job metadata: generateName: change-replication-password-on-secondary- + namespace: adhoc-jobs spec: template: + ttlSecondsAfterFinished: 604800 spec: ttlSecondsAfterFinished: 604800 containers: diff --git a/k8s/jobs/elasticSearchImportJob.yaml b/k8s/jobs/elasticSearchImportJob.yaml index fe38d63d6..bd4ffc413 100644 --- a/k8s/jobs/elasticSearchImportJob.yaml +++ b/k8s/jobs/elasticSearchImportJob.yaml @@ -2,8 +2,10 @@ apiVersion: batch/v1 kind: Job metadata: generateName: load-elasticsearch-data- + namespace: adhoc-jobs spec: template: + ttlSecondsAfterFinished: 604800 metadata: name: load-elasticsearch-data spec: diff --git a/k8s/jobs/forceSearchIndexFrom.yaml b/k8s/jobs/forceSearchIndexFrom.yaml index e8cad7ce9..bf68fc39d 100644 --- a/k8s/jobs/forceSearchIndexFrom.yaml +++ b/k8s/jobs/forceSearchIndexFrom.yaml @@ -2,8 +2,10 @@ apiVersion: batch/v1 kind: Job metadata: generateName: force-search-index-from- + namespace: adhoc-jobs spec: template: + ttlSecondsAfterFinished: 604800 metadata: name: force-search-index-from spec: diff --git a/k8s/jobs/rebuildQuantityUnitsJob.yaml b/k8s/jobs/rebuildQuantityUnitsJob.yaml index 6728f5dd0..a7d061e0a 100644 --- a/k8s/jobs/rebuildQuantityUnitsJob.yaml +++ b/k8s/jobs/rebuildQuantityUnitsJob.yaml @@ -2,8 +2,10 @@ apiVersion: batch/v1 kind: Job metadata: generateName: rebuild-quantity-units- + namespace: adhoc-jobs spec: template: + ttlSecondsAfterFinished: 604800 metadata: name: rebuild-quantity-units spec: diff --git a/k8s/jobs/resetOtherSqlSecretsJob.yaml b/k8s/jobs/resetOtherSqlSecretsJob.yaml index 11e0be3e5..42399d77e 100644 --- a/k8s/jobs/resetOtherSqlSecretsJob.yaml +++ b/k8s/jobs/resetOtherSqlSecretsJob.yaml @@ -2,8 +2,10 @@ apiVersion: batch/v1 kind: Job metadata: generateName: reset-other-sql-secrets-job- + namespace: adhoc-jobs spec: template: + ttlSecondsAfterFinished: 604800 spec: ttlSecondsAfterFinished: 604800 containers: diff --git a/k8s/jobs/resetRootSqlSecretJob.yaml b/k8s/jobs/resetRootSqlSecretJob.yaml index 3a4e2f64e..07e13be07 100644 --- a/k8s/jobs/resetRootSqlSecretJob.yaml +++ b/k8s/jobs/resetRootSqlSecretJob.yaml @@ -2,8 +2,10 @@ apiVersion: batch/v1 kind: Job metadata: generateName: reset-root-sql-secret-job- + namespace: adhoc-jobs spec: template: + ttlSecondsAfterFinished: 604800 spec: ttlSecondsAfterFinished: 604800 containers: diff --git a/k8s/jobs/runAllMWJobsJob.yaml b/k8s/jobs/runAllMWJobsJob.yaml index 3a1f8cb01..3ef06aba1 100644 --- a/k8s/jobs/runAllMWJobsJob.yaml +++ b/k8s/jobs/runAllMWJobsJob.yaml @@ -2,8 +2,10 @@ apiVersion: batch/v1 kind: Job metadata: generateName: run-all-mw-jobs- + namespace: adhoc-jobs spec: template: + ttlSecondsAfterFinished: 604800 metadata: name: run-all-mw-jobs spec: diff --git a/tf/env/local/namespaces.tf b/tf/env/local/namespaces.tf index a707249c4..518a5df19 100644 --- a/tf/env/local/namespaces.tf +++ b/tf/env/local/namespaces.tf @@ -16,3 +16,23 @@ resource "kubernetes_resource_quota" "api-jobs-podquota" { scopes = ["BestEffort"] } } + + +resource "kubernetes_namespace" "adhoc-job-namespace" { + metadata { + name = "adhoc-jobs" + } +} + +resource "kubernetes_resource_quota" "adhoc-jobs-podquota" { + metadata { + name = "api-jobs-podquota" + namespace = kubernetes_namespace.api-job-namespace.metadata[0].name + } + spec { + hard = { + pods = 1 + } + scopes = ["BestEffort"] + } +} diff --git a/tf/env/local/secrets-api.tf b/tf/env/local/secrets-api.tf index 76c912c5a..7ca8476e3 100644 --- a/tf/env/local/secrets-api.tf +++ b/tf/env/local/secrets-api.tf @@ -29,7 +29,7 @@ resource "random_password" "api-app-jwt-secret" { } resource "kubernetes_secret" "api-app-secrets" { - for_each = toset(["default", "api-jobs"]) + for_each = toset(["default", "api-jobs", "adhoc-jobs"]) metadata { name = "api-app-secrets" namespace = each.value diff --git a/tf/env/local/secrets-recapcha.tf b/tf/env/local/secrets-recapcha.tf index c0b819682..a6e44e1f1 100644 --- a/tf/env/local/secrets-recapcha.tf +++ b/tf/env/local/secrets-recapcha.tf @@ -1,5 +1,5 @@ resource "kubernetes_secret" "recaptcha-v3-dev-secrets" { - for_each = toset(["default", "api-jobs"]) + for_each = toset(["default", "api-jobs", "adhoc-jobs"]) metadata { name = "recaptcha-v3-dev-secrets" # default as staging @@ -18,7 +18,7 @@ moved { } resource "kubernetes_secret" "recaptcha-v2-dev-secrets" { - for_each = toset(["default", "api-jobs"]) + for_each = toset(["default", "api-jobs", "adhoc-jobs"]) metadata { name = "recaptcha-v2-dev-secrets" # default as staging diff --git a/tf/env/local/secrets-redis.tf b/tf/env/local/secrets-redis.tf index 08cd8c5ae..a77b4481e 100644 --- a/tf/env/local/secrets-redis.tf +++ b/tf/env/local/secrets-redis.tf @@ -7,7 +7,7 @@ resource "random_password" "redis-password" { # Used by the sql service for initial setup resource "kubernetes_secret" "redis-password" { - for_each = toset(["default", "api-jobs"]) + for_each = toset(["default", "api-jobs", "adhoc-jobs"]) metadata { name = "redis-password" namespace = each.value diff --git a/tf/env/local/secrets-sql.tf b/tf/env/local/secrets-sql.tf index 99060072d..d1778d6ba 100644 --- a/tf/env/local/secrets-sql.tf +++ b/tf/env/local/secrets-sql.tf @@ -8,7 +8,7 @@ resource "random_password" "sql-passwords" { # Used by the sql service for initial setup resource "kubernetes_secret" "sql-secrets-passwords" { - for_each = toset(["default", "api-jobs"]) + for_each = toset(["default", "api-jobs", "adhoc-jobs"]) metadata { name = "sql-secrets-passwords" namespace = each.value @@ -27,7 +27,7 @@ moved { # Used by the init script on sql services for user and permissions setup resource "kubernetes_secret" "sql-secrets-init-passwords" { - for_each = toset(["default", "api-jobs"]) + for_each = toset(["default", "api-jobs", "adhoc-jobs"]) metadata { name = "sql-secrets-init-passwords" namespace = each.value diff --git a/tf/env/production/kubernetes-secrets.tf b/tf/env/production/kubernetes-secrets.tf index 01312e3e9..7362677c2 100644 --- a/tf/env/production/kubernetes-secrets.tf +++ b/tf/env/production/kubernetes-secrets.tf @@ -21,7 +21,11 @@ module "wbaas-k8s-secrets" { api_passport_private_key = tls_private_key.api-passport.private_key_pem api_app_key = random_password.api-app-key.result api_app_jwt_secret = random_password.api-app-jwt-secret.result - mediawiki_secret_namespaces = ["default", kubernetes_namespace.api-job-namespace.metadata[0].name] + mediawiki_secret_namespaces = [ + "default", + kubernetes_namespace.api-job-namespace.metadata[0].name, + kubernetes_namespace.adhoc-job-namespace.metadata[0].name + ] logical_backup_openssl_secret = random_password.logical_backup_random_password.result } diff --git a/tf/env/production/namespaces.tf b/tf/env/production/namespaces.tf index 7b44860f3..b5f76b732 100644 --- a/tf/env/production/namespaces.tf +++ b/tf/env/production/namespaces.tf @@ -20,3 +20,27 @@ resource "kubernetes_resource_quota" "api-jobs-podquota" { scopes = ["BestEffort"] } } + + +resource "kubernetes_namespace" "adhoc-job-namespace" { + provider = kubernetes.wbaas-2 + + metadata { + name = "adhoc-jobs" + } +} + +resource "kubernetes_resource_quota" "adhoc-jobs-podquota" { + provider = kubernetes.wbaas-2 + + metadata { + name = "api-jobs-podquota" + namespace = kubernetes_namespace.api-job-namespace.metadata[0].name + } + spec { + hard = { + pods = 8 + } + scopes = ["BestEffort"] + } +} diff --git a/tf/env/staging/kubernetes-secrets.tf b/tf/env/staging/kubernetes-secrets.tf index 5052c6c2c..5dec99223 100644 --- a/tf/env/staging/kubernetes-secrets.tf +++ b/tf/env/staging/kubernetes-secrets.tf @@ -21,6 +21,10 @@ module "wbaas2-k8s-secrets" { api_passport_private_key = tls_private_key.api-passport.private_key_pem api_app_key = random_password.api-app-key.result api_app_jwt_secret = random_password.api-app-jwt-secret.result - mediawiki_secret_namespaces = ["default", kubernetes_namespace.api-job-namespace.metadata[0].name] - logical_backup_openssl_secret = random_password.logical_backup_random_password.result + mediawiki_secret_namespaces = [ + "default", + kubernetes_namespace.api-job-namespace.metadata[0].name, + kubernetes_namespace.adhoc-job-namespace.metadata[0].name + ] + logical_backup_openssl_secret = random_password.logical_backup_random_password.result } diff --git a/tf/env/staging/namespaces.tf b/tf/env/staging/namespaces.tf index b66efb794..cefaaa8f0 100644 --- a/tf/env/staging/namespaces.tf +++ b/tf/env/staging/namespaces.tf @@ -20,3 +20,26 @@ resource "kubernetes_resource_quota" "api-jobs-podquota" { scopes = ["BestEffort"] } } + +resource "kubernetes_namespace" "adhoc-job-namespace" { + provider = kubernetes.wbaas-2 + + metadata { + name = "adhoc-jobs" + } +} + +resource "kubernetes_resource_quota" "adhoc-jobs-podquota" { + provider = kubernetes.wbaas-2 + + metadata { + name = "api-jobs-podquota" + namespace = kubernetes_namespace.api-job-namespace.metadata[0].name + } + spec { + hard = { + pods = 4 + } + scopes = ["BestEffort"] + } +}