Implement nginx based caching layer

tf/modules/nginx_reverseproxy/main.tf

@@ -0,0 +1,110 @@
+data "aws_ssm_parameter" "ubuntu_22_ami" {
+  name = "/aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id"
+}
+
+# Important note about security groups:
+# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group#recreating-a-security-group
+resource "aws_security_group" "nginx" {
+  description = "security group for nginx"
+
+  vpc_id = var.vpc_id
+
+  ingress {
+    protocol    = "tcp"
+    from_port   = 80
+    to_port     = 80
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    protocol    = "tcp"
+    from_port   = 22
+    to_port     = 22
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port = 0
+    to_port   = 0
+    protocol  = "-1"
+
+    cidr_blocks = [
+      "0.0.0.0/0",
+    ]
+  }
+
+  tags = var.tags
+}
+
+resource "aws_launch_template" "nginx" {
+  name_prefix   = "${var.name}-nginx-tmpl-"
+  image_id      = data.aws_ssm_parameter.ubuntu_22_ami.value
+  instance_type = var.instance_type
+  key_name      = var.key_name
+
+  user_data = base64encode(templatefile("${path.module}/templates/setup-reverse-proxy.sh", {
+    proxy_pass_url     = var.proxy_pass_url,
+    extra_path_config  = var.nginx_extra_path_config,
+    extra_nginx_config = var.nginx_extra_nginx_config,
+  }))
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  network_interfaces {
+    delete_on_termination       = true
+    associate_public_ip_address = true
+    security_groups = [
+      aws_security_group.nginx.id,
+    ]
+  }
+
+  tag_specifications {
+    resource_type = "instance"
+    tags          = var.tags
+  }
+}
+
+resource "aws_autoscaling_group" "nginx" {
+  launch_template {
+    id      = aws_launch_template.nginx.id
+    version = "$Latest"
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  name_prefix = "${var.name}-asg-"
+
+  min_size            = 1
+  max_size            = 2
+  desired_capacity    = 1
+  vpc_zone_identifier = var.subnet_ids
+
+  instance_refresh {
+    strategy = "Rolling"
+    preferences {
+      min_healthy_percentage = 50
+    }
+  }
+}
+
+resource "aws_alb_target_group" "nginx" {
+  name     = var.name
+  port     = 80
+  protocol = "HTTP"
+  vpc_id   = var.vpc_id
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  tags = var.tags
+}
+
+resource "aws_autoscaling_attachment" "nginx" {
+  autoscaling_group_name = aws_autoscaling_group.nginx.id
+  lb_target_group_arn    = aws_alb_target_group.nginx.arn
+}

tf/modules/nginx_reverseproxy/outputs.tf

@@ -0,0 +1,6 @@
+output "autoscaling_group_id" {
+  value = aws_autoscaling_group.nginx.id
+}
+output "alb_target_group_id" {
+  value = aws_alb_target_group.nginx.id
+}

tf/modules/nginx_reverseproxy/templates/setup-reverse-proxy.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+set -e
+
+sudo apt update
+sudo apt install -y nginx
+
+tmpfile=$(mktemp /tmp/nginx-config.XXXXXX)
+cat > $tmpfile <<EOF
+server {
+    listen 80;
+
+    server_name _;
+
+    location / {
+        proxy_pass ${proxy_pass_url};
+        proxy_http_version 1.1;
+        proxy_set_header Host \$host;
+
+        ${extra_path_config}
+    }
+}
+EOF
+sudo mv $tmpfile /etc/nginx/sites-available/default
+
+tmpfile=$(mktemp /tmp/nginx-config.XXXXXX)
+cat > $tmpfile <<EOF
+user www-data;
+worker_processes auto;
+pid /run/nginx.pid;
+include /etc/nginx/modules-enabled/*.conf;
+
+events {
+  worker_connections 768;
+}
+
+http {
+  sendfile on;
+  tcp_nopush on;
+  types_hash_max_size 2048;
+  include /etc/nginx/mime.types;
+  default_type application/octet-stream;
+
+  map \$remote_addr \$remote_addr_anon {
+    ~(?P<ip>\\d+\\.\\d+\.\\d+)\.    \$ip.0;
+    ~(?P<ip>[^:]+:[^:]+):       \$ip::;
+    default                     0.0.0.0;
+  }
+
+  # log anonymized ipaddr and caching status
+  log_format ooni_nginx_fmt '\$remote_addr_anon \$upstream_cache_status [\$time_local] '
+      '"\$request" \$status \$body_bytes_sent "\$http_referer" "\$http_user_agent"';
+
+  ${extra_nginx_config}
+
+  access_log syslog:server=unix:/dev/log ooni_nginx_fmt;
+  error_log syslog:server=unix:/dev/log;
+
+  gzip on;
+
+  include /etc/nginx/conf.d/*.conf;
+  include /etc/nginx/sites-enabled/*;
+}
+EOF
+sudo mv $tmpfile /etc/nginx/nginx.conf
+
+sudo mkdir -p /var/cache/nginx
+sudo chown -R www-data /var/cache/nginx
+
+sudo nginx -t
+sudo systemctl reload nginx

tf/modules/nginx_reverseproxy/variables.tf

@@ -0,0 +1,40 @@
+variable "vpc_id" {
+  description = "the id of the VPC to deploy the instance into"
+}
+
+variable "subnet_ids" {
+  description = "the ids of the subnet of the subnets to deploy the instance into"
+}
+
+variable "tags" {
+  description = "tags to apply to the resources"
+  default     = {}
+  type        = map(string)
+}
+
+variable "key_name" {
+  description = "Name of AWS key pair"
+}
+
+variable "name" {
+  description = "Name of the resources"
+  default     = "ooni-backendproxy"
+}
+
+variable "instance_type" {
+  default = "t2.micro"
+}
+
+
+variable "proxy_pass_url" {
+  description = "URL to pass to the proxy_pass directive"
+}
+
+variable "nginx_extra_nginx_config" {
+  description = "extra configuration to pass to nginx"
+  default     = ""
+}
+variable "nginx_extra_path_config" {
+  description = "extra configuration to pass to nginx"
+  default     = ""
+}

tf/modules/oonith_service/main.tf

@@ -7,6 +7,7 @@ locals {
   short_prefix = "oo${substr(var.service_name, 0, 3)}"
 }
 
+
 resource "aws_iam_role" "oonith_service_task" {
   name = "${local.name}-task-role"
 
@@ -188,7 +189,7 @@ resource "aws_alb_target_group" "oonith_service_direct" {
 # }
 
 resource "aws_alb" "oonith_service" {
-  name            = local.name
+  name_prefix     = "ooth"
   subnets         = var.public_subnet_ids
   security_groups = var.oonith_service_security_groups
 
@@ -208,15 +209,64 @@ resource "aws_alb_listener" "oonith_service_http" {
   tags = var.tags
 }
 
+module "oonith_nginx_cache" {
+  source = "../nginx_reverseproxy"
+
+  vpc_id        = var.vpc_id
+  subnet_ids    = var.public_subnet_ids
+  key_name      = var.key_name
+  instance_type = "t2.micro"
+  tags          = var.tags
+
+  name                     = "oonith-nginx-cache"
+  proxy_pass_url           = "http://${aws_alb.oonith_service.dns_name}/"
+  nginx_extra_path_config  = <<EOT
+      proxy_cache thcache;
+      proxy_cache_min_uses 1;
+      proxy_cache_lock on;
+      proxy_cache_lock_timeout 30;
+      proxy_cache_lock_age 30;
+      proxy_cache_use_stale error timeout invalid_header updating;
+      # Cache POST without headers set by the test helper!
+      proxy_cache_methods POST;
+      proxy_cache_key "\$request_uri|\$request_body";
+      proxy_cache_valid 200 10m;
+      proxy_cache_valid any 0;
+      add_header X-Cache-Status \$upstream_cache_status;
+      EOT
+  nginx_extra_nginx_config = "proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M max_size=5g inactive=24h use_temp_path=off;"
+}
+
+resource "aws_alb" "front_end" {
+  name_prefix     = "front"
+  subnets         = var.public_subnet_ids
+  security_groups = var.oonith_service_security_groups
+
+  tags = var.tags
+}
+
+resource "aws_alb_listener" "front_end_http" {
+  load_balancer_arn = aws_alb.front_end.id
+  port              = "80"
+  protocol          = "HTTP"
+
+  default_action {
+    target_group_arn = module.oonith_nginx_cache.alb_target_group_id
+    type             = "forward"
+  }
+
+  tags = var.tags
+}
+
 resource "aws_alb_listener" "front_end_https" {
-  load_balancer_arn = aws_alb.oonith_service.id
+  load_balancer_arn = aws_alb.front_end.id
   port              = "443"
   protocol          = "HTTPS"
   ssl_policy        = "ELBSecurityPolicy-2016-08"
   certificate_arn   = aws_acm_certificate.oonith_service.arn
 
   default_action {
-    target_group_arn = aws_alb_target_group.oonith_service_direct.id
+    target_group_arn = module.oonith_nginx_cache.alb_target_group_id
     type             = "forward"
   }
 
@@ -229,8 +279,8 @@ resource "aws_route53_record" "oonith_service" {
   type    = "A"
 
   alias {
-    name                   = aws_alb.oonith_service.dns_name
-    zone_id                = aws_alb.oonith_service.zone_id
+    name                   = aws_alb.front_end.dns_name
+    zone_id                = aws_alb.front_end.zone_id
     evaluate_target_health = true
   }
 }
@@ -283,8 +333,8 @@ resource "aws_route53_record" "oonith_service_alias" {
   type    = "A"
 
   alias {
-    name                   = aws_alb.oonith_service.dns_name
-    zone_id                = aws_alb.oonith_service.zone_id
+    name                   = aws_alb.front_end.dns_name
+    zone_id                = aws_alb.front_end.zone_id
     evaluate_target_health = true
   }
 }