From 415a8a6da6282a6f8f5402da09ff2ffb568b101a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Fri, 19 Apr 2024 19:15:02 +0200
Subject: [PATCH 1/3] Implement nginx based caching layer

---
 tf/modules/nginx_reverseproxy/main.tf         | 109 ++++++++++++++++++
 tf/modules/nginx_reverseproxy/outputs.tf      |   6 +
 .../templates/setup-reverse-proxy.sh          |  26 +++++
 tf/modules/nginx_reverseproxy/variables.tf    |  36 ++++++
 tf/modules/oonith_service/main.tf             |  63 ++++++++--
 5 files changed, 233 insertions(+), 7 deletions(-)
 create mode 100644 tf/modules/nginx_reverseproxy/main.tf
 create mode 100644 tf/modules/nginx_reverseproxy/outputs.tf
 create mode 100644 tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh
 create mode 100644 tf/modules/nginx_reverseproxy/variables.tf

diff --git a/tf/modules/nginx_reverseproxy/main.tf b/tf/modules/nginx_reverseproxy/main.tf
new file mode 100644
index 00000000..82357c4e
--- /dev/null
+++ b/tf/modules/nginx_reverseproxy/main.tf
@@ -0,0 +1,109 @@
+data "aws_ssm_parameter" "ubuntu_22_ami" {
+  name = "/aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id"
+}
+
+# Important note about security groups:
+# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group#recreating-a-security-group
+resource "aws_security_group" "nginx" {
+  description = "security group for nginx"
+
+  vpc_id = var.vpc_id
+
+  ingress {
+    protocol    = "tcp"
+    from_port   = 80
+    to_port     = 80
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    protocol    = "tcp"
+    from_port   = 22
+    to_port     = 22
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+
+    cidr_blocks = [
+      "0.0.0.0/0",
+    ]
+  }
+
+  tags = var.tags
+}
+
+resource "aws_launch_template" "nginx" {
+  name_prefix   = "${var.name}-nginx-tmpl-"
+  image_id      = data.aws_ssm_parameter.ubuntu_22_ami.value
+  instance_type = var.instance_type
+  key_name      = var.key_name
+
+  user_data = base64encode(templatefile("${path.module}/templates/setup-reverse-proxy.sh", {
+    proxy_pass_url = var.proxy_pass_url,
+    extra_config   = var.nginx_extra_config,
+  }))
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  network_interfaces {
+    delete_on_termination       = true
+    associate_public_ip_address = true
+    security_groups = [
+      aws_security_group.nginx.id,
+    ]
+  }
+
+  tag_specifications {
+    resource_type = "instance"
+    tags          = var.tags
+  }
+}
+
+resource "aws_autoscaling_group" "nginx" {
+  launch_template {
+    id      = aws_launch_template.nginx.id
+    version = "$Latest"
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  name_prefix = "${var.name}-asg-"
+
+  min_size            = 1
+  max_size            = 2
+  desired_capacity    = 1
+  vpc_zone_identifier = var.subnet_ids
+
+  instance_refresh {
+    strategy = "Rolling"
+    preferences {
+      min_healthy_percentage = 50
+    }
+  }
+}
+
+resource "aws_alb_target_group" "nginx" {
+  name     = var.name
+  port     = 80
+  protocol = "HTTP"
+  vpc_id   = var.vpc_id
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  tags = var.tags
+}
+
+resource "aws_autoscaling_attachment" "nginx" {
+  autoscaling_group_name = aws_autoscaling_group.nginx.id
+  lb_target_group_arn    = aws_alb_target_group.nginx.arn
+}
diff --git a/tf/modules/nginx_reverseproxy/outputs.tf b/tf/modules/nginx_reverseproxy/outputs.tf
new file mode 100644
index 00000000..aac0d3e1
--- /dev/null
+++ b/tf/modules/nginx_reverseproxy/outputs.tf
@@ -0,0 +1,6 @@
+output "autoscaling_group_id" {
+  value = aws_autoscaling_group.nginx.id
+}
+output "alb_target_group_id" {
+  value = aws_alb_target_group.nginx.id
+}
diff --git a/tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh b/tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh
new file mode 100644
index 00000000..2a762a40
--- /dev/null
+++ b/tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+sudo apt update
+sudo apt install -y nginx
+
+tmpfile=$(mktemp /tmp/nginx-config.XXXXXX)
+cat > $tmpfile <<EOF
+server {
+    listen 80;
+
+    server_name _;
+
+    location / {
+        proxy_pass ${proxy_pass_url};
+        proxy_http_version 1.1;
+        proxy_set_header Host \$host;
+
+        ${extra_config}
+    }
+}
+EOF
+sudo mv $tmpfile /etc/nginx/sites-available/default
+
+sudo nginx -t
+sudo systemctl reload nginx
diff --git a/tf/modules/nginx_reverseproxy/variables.tf b/tf/modules/nginx_reverseproxy/variables.tf
new file mode 100644
index 00000000..5f2cc653
--- /dev/null
+++ b/tf/modules/nginx_reverseproxy/variables.tf
@@ -0,0 +1,36 @@
+variable "vpc_id" {
+  description = "the id of the VPC to deploy the instance into"
+}
+
+variable "subnet_ids" {
+  description = "the ids of the subnet of the subnets to deploy the instance into"
+}
+
+variable "tags" {
+  description = "tags to apply to the resources"
+  default     = {}
+  type        = map(string)
+}
+
+variable "key_name" {
+  description = "Name of AWS key pair"
+}
+
+variable "name" {
+  description = "Name of the resources"
+  default     = "ooni-backendproxy"
+}
+
+variable "instance_type" {
+  default = "t2.micro"
+}
+
+
+variable "proxy_pass_url" {
+  description = "URL to pass to the proxy_pass directive"
+}
+
+variable "nginx_extra_config" {
+  description = "extra configuration to pass to nginx"
+  default     = ""
+}
diff --git a/tf/modules/oonith_service/main.tf b/tf/modules/oonith_service/main.tf
index 681016a3..f7b9512f 100644
--- a/tf/modules/oonith_service/main.tf
+++ b/tf/modules/oonith_service/main.tf
@@ -7,6 +7,7 @@ locals {
   short_prefix = "oo${substr(var.service_name, 0, 3)}"
 }
 
+
 resource "aws_iam_role" "oonith_service_task" {
   name = "${local.name}-task-role"
 
@@ -188,7 +189,7 @@ resource "aws_alb_target_group" "oonith_service_direct" {
 # }
 
 resource "aws_alb" "oonith_service" {
-  name            = local.name
+  name_prefix     = "ooth"
   subnets         = var.public_subnet_ids
   security_groups = var.oonith_service_security_groups
 
@@ -208,15 +209,63 @@ resource "aws_alb_listener" "oonith_service_http" {
   tags = var.tags
 }
 
+module "oonith_nginx_cache" {
+  source = "../nginx_reverseproxy"
+
+  vpc_id        = var.vpc_id
+  subnet_ids    = var.public_subnet_ids
+  key_name      = var.key_name
+  instance_type = "t2.micro"
+  tags          = var.tags
+
+  name               = "oonith-nginx-cache"
+  proxy_pass_url     = "http://${aws_alb.oonith_service.dns_name}/"
+  nginx_extra_config = <<EOT
+      proxy_cache thcache;
+      proxy_cache_min_uses 1;
+      proxy_cache_lock on;
+      proxy_cache_lock_timeout 30;
+      proxy_cache_lock_age 30;
+      proxy_cache_use_stale error timeout invalid_header updating;
+      # Cache POST without headers set by the test helper!
+      proxy_cache_methods POST;
+      proxy_cache_key "$request_uri|$request_body";
+      proxy_cache_valid 200 10m;
+      proxy_cache_valid any 0;
+      add_header X-Cache-Status $upstream_cache_status;
+      EOT
+}
+
+resource "aws_alb" "front_end" {
+  name_prefix     = "front"
+  subnets         = var.public_subnet_ids
+  security_groups = var.oonith_service_security_groups
+
+  tags = var.tags
+}
+
+resource "aws_alb_listener" "front_end_http" {
+  load_balancer_arn = aws_alb.front_end.id
+  port              = "80"
+  protocol          = "HTTP"
+
+  default_action {
+    target_group_arn = module.oonith_nginx_cache.alb_target_group_id
+    type             = "forward"
+  }
+
+  tags = var.tags
+}
+
 resource "aws_alb_listener" "front_end_https" {
-  load_balancer_arn = aws_alb.oonith_service.id
+  load_balancer_arn = aws_alb.front_end.id
   port              = "443"
   protocol          = "HTTPS"
   ssl_policy        = "ELBSecurityPolicy-2016-08"
   certificate_arn   = aws_acm_certificate.oonith_service.arn
 
   default_action {
-    target_group_arn = aws_alb_target_group.oonith_service_direct.id
+    target_group_arn = module.oonith_nginx_cache.alb_target_group_id
     type             = "forward"
   }
 
@@ -229,8 +278,8 @@ resource "aws_route53_record" "oonith_service" {
   type    = "A"
 
   alias {
-    name                   = aws_alb.oonith_service.dns_name
-    zone_id                = aws_alb.oonith_service.zone_id
+    name                   = aws_alb.front_end.dns_name
+    zone_id                = aws_alb.front_end.zone_id
     evaluate_target_health = true
   }
 }
@@ -283,8 +332,8 @@ resource "aws_route53_record" "oonith_service_alias" {
   type    = "A"
 
   alias {
-    name                   = aws_alb.oonith_service.dns_name
-    zone_id                = aws_alb.oonith_service.zone_id
+    name                   = aws_alb.front_end.dns_name
+    zone_id                = aws_alb.front_end.zone_id
     evaluate_target_health = true
   }
 }

From 740664a2557884d498f6022708bf417fe1835255 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Fri, 19 Apr 2024 20:08:36 +0200
Subject: [PATCH 2/3] Set caching config

---
 tf/modules/nginx_reverseproxy/main.tf         |  5 +-
 .../templates/setup-reverse-proxy.sh          | 46 ++++++++++++++++++-
 tf/modules/nginx_reverseproxy/variables.tf    |  6 ++-
 tf/modules/oonith_service/main.tf             | 11 +++--
 4 files changed, 59 insertions(+), 9 deletions(-)

diff --git a/tf/modules/nginx_reverseproxy/main.tf b/tf/modules/nginx_reverseproxy/main.tf
index 82357c4e..2dcdfeeb 100644
--- a/tf/modules/nginx_reverseproxy/main.tf
+++ b/tf/modules/nginx_reverseproxy/main.tf
@@ -43,8 +43,9 @@ resource "aws_launch_template" "nginx" {
   key_name      = var.key_name
 
   user_data = base64encode(templatefile("${path.module}/templates/setup-reverse-proxy.sh", {
-    proxy_pass_url = var.proxy_pass_url,
-    extra_config   = var.nginx_extra_config,
+    proxy_pass_url     = var.proxy_pass_url,
+    extra_path_config  = var.nginx_extra_path_config,
+    extra_nginx_config = var.nginx_extra_nginx_config,
   }))
 
   lifecycle {
diff --git a/tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh b/tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh
index 2a762a40..448a3ef6 100644
--- a/tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh
+++ b/tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh
@@ -16,11 +16,55 @@ server {
         proxy_http_version 1.1;
         proxy_set_header Host \$host;
 
-        ${extra_config}
+        ${extra_path_config}
     }
 }
 EOF
 sudo mv $tmpfile /etc/nginx/sites-available/default
 
+tmpfile=$(mktemp /tmp/nginx-config.XXXXXX)
+cat > $tmpfile <<EOF
+user www-data;
+worker_processes auto;
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+  worker_connections 768;
+}
+
+http {
+  sendfile on;
+  tcp_nopush on;
+  types_hash_max_size 2048;
+  include /etc/nginx/mime.types;
+  default_type application/octet-stream;
+
+  map \$remote_addr \$remote_addr_anon {
+    ~(?P<ip>\\d+\\.\\d+\.\\d+)\.    \$ip.0;
+    ~(?P<ip>[^:]+:[^:]+):       \$ip::;
+    default                     0.0.0.0;
+  }
+
+  # log anonymized ipaddr and caching status
+  log_format ooni_nginx_fmt '\$remote_addr_anon \$upstream_cache_status [\$time_local] '
+      '"\$request" \$status \$body_bytes_sent "\$http_referer" "\$http_user_agent"';
+
+  ${extra_nginx_config}
+
+  access_log syslog:server=unix:/dev/log ooni_nginx_fmt;
+  error_log syslog:server=unix:/dev/log;
+
+  gzip on;
+
+  include /etc/nginx/conf.d/*.conf;
+  include /etc/nginx/sites-enabled/*;
+}
+EOF
+sudo mv $tmpfile /etc/nginx/sites-available/default
+
+sudo mkdir -p /var/cache/nginx
+sudo chown -R www-data /var/cache/nginx
+
 sudo nginx -t
 sudo systemctl reload nginx
diff --git a/tf/modules/nginx_reverseproxy/variables.tf b/tf/modules/nginx_reverseproxy/variables.tf
index 5f2cc653..8fbf6f62 100644
--- a/tf/modules/nginx_reverseproxy/variables.tf
+++ b/tf/modules/nginx_reverseproxy/variables.tf
@@ -30,7 +30,11 @@ variable "proxy_pass_url" {
   description = "URL to pass to the proxy_pass directive"
 }
 
-variable "nginx_extra_config" {
+variable "nginx_extra_nginx_config" {
+  description = "extra configuration to pass to nginx"
+  default     = ""
+}
+variable "nginx_extra_path_config" {
   description = "extra configuration to pass to nginx"
   default     = ""
 }
diff --git a/tf/modules/oonith_service/main.tf b/tf/modules/oonith_service/main.tf
index f7b9512f..497e1ce6 100644
--- a/tf/modules/oonith_service/main.tf
+++ b/tf/modules/oonith_service/main.tf
@@ -218,9 +218,9 @@ module "oonith_nginx_cache" {
   instance_type = "t2.micro"
   tags          = var.tags
 
-  name               = "oonith-nginx-cache"
-  proxy_pass_url     = "http://${aws_alb.oonith_service.dns_name}/"
-  nginx_extra_config = <<EOT
+  name                     = "oonith-nginx-cache"
+  proxy_pass_url           = "http://${aws_alb.oonith_service.dns_name}/"
+  nginx_extra_path_config  = <<EOT
       proxy_cache thcache;
       proxy_cache_min_uses 1;
       proxy_cache_lock on;
@@ -229,11 +229,12 @@ module "oonith_nginx_cache" {
       proxy_cache_use_stale error timeout invalid_header updating;
       # Cache POST without headers set by the test helper!
       proxy_cache_methods POST;
-      proxy_cache_key "$request_uri|$request_body";
+      proxy_cache_key "\$request_uri|\$request_body";
       proxy_cache_valid 200 10m;
       proxy_cache_valid any 0;
-      add_header X-Cache-Status $upstream_cache_status;
+      add_header X-Cache-Status \$upstream_cache_status;
       EOT
+  nginx_extra_nginx_config = "proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M max_size=5g inactive=24h use_temp_path=off;"
 }
 
 resource "aws_alb" "front_end" {

From cc6aa625d33009eff796f2447462ba3b843bd116 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Arturo=20Filast=C3=B2?= <arturo@filasto.net>
Date: Fri, 19 Apr 2024 20:21:56 +0200
Subject: [PATCH 3/3] Fix template move

---
 tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh b/tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh
index 448a3ef6..18e10d15 100644
--- a/tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh
+++ b/tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh
@@ -61,7 +61,7 @@ http {
   include /etc/nginx/sites-enabled/*;
 }
 EOF
-sudo mv $tmpfile /etc/nginx/sites-available/default
+sudo mv $tmpfile /etc/nginx/nginx.conf
 
 sudo mkdir -p /var/cache/nginx
 sudo chown -R www-data /var/cache/nginx