From 06836ad5b6fc59575a3afc393e032428c4b6c5b2 Mon Sep 17 00:00:00 2001
From: Lens0021 / Leslie <lorentz0021@gmail.com>
Date: Sun, 6 Oct 2024 18:06:33 +0900
Subject: [PATCH] Use EBS for Caddy on the green cluster (#298)

---
 jobs/http.nomad        | 87 ++++++++----------------------------------
 terraform/mediawiki.tf |  8 ++--
 terraform/volumes.tf   | 38 ++++++++++++++++++
 3 files changed, 57 insertions(+), 76 deletions(-)

diff --git a/jobs/http.nomad b/jobs/http.nomad
index d6ffbfd9..9f396659 100644
--- a/jobs/http.nomad
+++ b/jobs/http.nomad
@@ -17,48 +17,29 @@ job "http" {
   datacenters = ["dc1"]
 
   group "http" {
-    dynamic "volume" {
-      for_each = local.main ? [{}] : []
-      labels   = ["caddycerts"]
-      content {
-        type            = "csi"
-        source          = "caddycerts"
-        read_only       = false
-        access_mode     = "single-node-writer"
-        attachment_mode = "file-system"
-      }
+    volume "caddycerts" {
+      type            = "csi"
+      source          = local.main ? "caddycerts" : "caddycerts_green"
+      read_only       = false
+      access_mode     = "single-node-writer"
+      attachment_mode = "file-system"
     }
 
     task "http" {
       driver = "docker"
 
-      dynamic "volume_mount" {
-        for_each = local.main ? [{}] : []
-        content {
-          volume      = "caddycerts"
-          destination = "/etc/caddycerts"
-          read_only   = false
-        }
+      volume_mount {
+        volume      = "caddycerts"
+        destination = "/etc/caddycerts"
+        read_only   = false
       }
 
-      dynamic "artifact" {
-        for_each = local.main ? [{}] : []
-
-        content {
-          source      = "https://github.com/femiwiki/nomad/raw/main/caddy/Caddyfile"
-          destination = "local/Caddyfile"
-          mode        = "file"
-
-          options { checksum = "md5:ee0300e384afa6aca74f09a44323ee6e" }
-        }
-      }
-      dynamic "template" {
-        for_each = var.test ? [{}] : []
+      artifact {
+        source      = "https://github.com/femiwiki/nomad/raw/main/caddy/Caddyfile"
+        destination = "local/Caddyfile"
+        mode        = "file"
 
-        content {
-          data        = var.caddyfile_for_test
-          destination = "local/Caddyfile"
-        }
+        options { checksum = "md5:ee0300e384afa6aca74f09a44323ee6e" }
       }
 
       artifact {
@@ -184,41 +165,3 @@ job "http" {
     auto_revert = true
   }
 }
-
-variable "caddyfile_for_test" {
-  type    = string
-  default = <<EOF
-{
-  # Global options
-  auto_https off
-  order mwcache before rewrite
-}
-http://127.0.0.1:80 http://localhost:80 http://{$TEST_NOMAD_PUBLIC_IP}:80
-root * /srv/femiwiki.com
-php_fastcgi {$FASTCGI_ADDR}
-file_server
-encode gzip
-mwcache {
-	ristretto {
-		num_counters 100000
-		max_cost 10000
-		buffer_items 64
-	}
-  purge_acl {
-    10.0.0.0/8
-    127.0.0.1
-  }
-}
-header {
-  # Enable XSS filtering for legacy browsers
-  X-XSS-Protection "1; mode=block"
-  # Block content sniffing, and enable Cross-Origin Read Blocking
-  X-Content-Type-Options "nosniff"
-  # Avoid clickjacking
-  X-Frame-Options "DENY"
-}
-rewrite /w/api.php /api.php
-rewrite /w/* /index.php
-
-EOF
-}
diff --git a/terraform/mediawiki.tf b/terraform/mediawiki.tf
index 2a6bab90..4505f267 100644
--- a/terraform/mediawiki.tf
+++ b/terraform/mediawiki.tf
@@ -86,10 +86,10 @@ resource "nomad_job" "http" {
 resource "nomad_job" "test_http" {
   provider = nomad.test
   # TODO Replace EBS CSI with S3 CSI or something
-  # depends_on = [
-  #   data.nomad_plugin.ebs,
-  #   nomad_csi_volume_registration.caddycerts,
-  # ]
+  depends_on = [
+    data.nomad_plugin.ebs_green,
+    nomad_csi_volume_registration.caddycerts_green,
+  ]
 
   jobspec = file("../jobs/http.nomad")
   detach  = false
diff --git a/terraform/volumes.tf b/terraform/volumes.tf
index 0bf06fdf..b8160236 100644
--- a/terraform/volumes.tf
+++ b/terraform/volumes.tf
@@ -14,11 +14,35 @@ resource "nomad_job" "plugin-ebs-nodes" {
   }
 }
 
+resource "nomad_job" "plugin_ebs_controller_green" {
+  provider = nomad.test
+  jobspec  = file("../jobs/plugin-ebs-controller.nomad")
+
+  hcl2 {
+    allow_fs = true
+  }
+}
+
+resource "nomad_job" "plugin_ebs_nodes_green" {
+  provider = nomad.test
+  jobspec  = file("../jobs/plugin-ebs-nodes.nomad")
+
+  hcl2 {
+    allow_fs = true
+  }
+}
+
 data "nomad_plugin" "ebs" {
   plugin_id        = "aws-ebs0"
   wait_for_healthy = true
 }
 
+data "nomad_plugin" "ebs_green" {
+  provider         = nomad.test
+  plugin_id        = "aws-ebs0"
+  wait_for_healthy = true
+}
+
 import {
   id = "mysql@default"
   to = nomad_csi_volume_registration.mysql
@@ -52,3 +76,17 @@ resource "nomad_csi_volume_registration" "caddycerts" {
     attachment_mode = "file-system"
   }
 }
+
+resource "nomad_csi_volume_registration" "caddycerts_green" {
+  provider    = nomad.test
+  depends_on  = [data.nomad_plugin.ebs_green]
+  plugin_id   = "aws-ebs0"
+  volume_id   = "caddycerts_green"
+  name        = "caddycerts_green"
+  external_id = data.terraform_remote_state.aws.outputs.ebs_caddycerts_green_id
+
+  capability {
+    access_mode     = "single-node-writer"
+    attachment_mode = "file-system"
+  }
+}