From 64bef613f9e3d9c80f2ceca5b5eed0fed4c391b4 Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 9 Jan 2025 00:17:44 +0530 Subject: [PATCH 1/4] refactor: add generic ec2 instance config --- tf/environments/dev/main.tf | 52 ++++++-- tf/modules/ec2/main.tf | 111 ++++++++++++++++++ tf/modules/ec2/outputs.tf | 7 ++ tf/modules/ec2/templates/cloud-init.yml | 0 tf/modules/ec2/variables.tf | 65 ++++++++++ tf/modules/ooni_backendproxy/main.tf | 1 - .../templates/cloud-init.yml | 58 --------- 7 files changed, 227 insertions(+), 67 deletions(-) create mode 100644 tf/modules/ec2/main.tf create mode 100644 tf/modules/ec2/outputs.tf create mode 100644 tf/modules/ec2/templates/cloud-init.yml create mode 100644 tf/modules/ec2/variables.tf diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index c7c32937..7cb2fbfc 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -414,8 +414,8 @@ module "ooniapi_reverseproxy" { ) } -module "ooni_backendproxy" { - source = "../../modules/ooni_backendproxy" +module "ooni_clickhouse_proxy" { + source = "../../modules/ec2" stage = local.environment @@ -427,19 +427,55 @@ module "ooni_backendproxy" { key_name = module.adm_iam_roles.oonidevops_key_name instance_type = "t3a.nano" - backend_url = "https://backend-fsn.ooni.org/" - wcth_addresses = module.ooni_th_droplet.droplet_ipv4_address - wcth_domain_suffix = "th.ooni.org" - clickhouse_url = "clickhouse1.prod.ooni.io" - clickhouse_port = "9000" + name = "oonickprx" + ingress_rules = [{ + from_port = 22, + to_port = 22, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 80, + to_port = 80, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 9000, + to_port = 9000, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }] + + egress_rules = [{ + from_port = 0, + to_port = 0, + protocol = "-1", + cidr_blocks = ["0.0.0.0/0"], + }, { + from_port = 0, + to_port = 0, + protocol = "-1", + ipv6_cidr_blocks = ["::/0"] + }] + + sg_prefix = "oockprx" + tg_prefix = "ckpr" tags = merge( local.tags, - { Name = "ooni-tier0-backendproxy" } + { Name = "ooni-tier0-clickhouseproxy" } ) } +resource "aws_route53_record" "clickhouse_proxy_alias" { + zone_id = local.dns_zone_ooni_io + name = "clickhouseproxy.${local.environment}.ooni.io" + type = "CNAME" + ttl = 300 + records = [ + module.ooni_clickhouse_proxy.aws_instance_public_dns + ] +} #### OONI Run service diff --git a/tf/modules/ec2/main.tf b/tf/modules/ec2/main.tf new file mode 100644 index 00000000..1e7691f0 --- /dev/null +++ b/tf/modules/ec2/main.tf @@ -0,0 +1,111 @@ +data "aws_ssm_parameter" "ubuntu_22_ami" { + name = "/aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id" +} + +# Important note about security groups: +# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group#recreating-a-security-group +resource "aws_security_group" "ec2_sg" { + description = "security group for ec2" + name_prefix = var.sg_prefix + + vpc_id = var.vpc_id + + lifecycle { + create_before_destroy = true + } + + tags = var.tags +} + +resource "aws_security_group_rule" "ec2_sg_ingress" { + count = length(var.ingress_rules) + + type = "ingress" + from_port = var.ingress_rules[count.index].from_port + to_port = var.ingress_rules[count.index].to_port + protocol = var.ingress_rules[count.index].protocol + cidr_blocks = var.ingress_rules[count.index].cidr_blocks + ipv6_cidr_blocks = var.ingress_rules[count.index].ipv6_cidr_blocks + security_group_id = aws_security_group.ec2_sg.id +} + +resource "aws_security_group_rule" "ec2_sg_egress" { + count = length(var.egress_rules) + + type = "egress" + from_port = var.egress_rules[count.index].from_port + to_port = var.egress_rules[count.index].to_port + protocol = var.egress_rules[count.index].protocol + cidr_blocks = var.egress_rules[count.index].cidr_blocks + ipv6_cidr_blocks = var.egress_rules[count.index].ipv6_cidr_blocks + security_group_id = aws_security_group.ec2_sg.id +} + +data "cloudinit_config" "ooni_ec2" { + base64_encode = true + + part { + filename = "init.cfg" + content_type = "text/cloud-config" + content = templatefile("${path.module}/templates/cloud-init.yml", {}) + } + +} + +resource "aws_launch_template" "ooni_ec2" { + name_prefix = "${var.name}-tmpl-" + image_id = data.aws_ssm_parameter.ubuntu_22_ami.value + instance_type = var.instance_type + key_name = var.key_name + + user_data = data.cloudinit_config.ooni_ec2.rendered + + lifecycle { + create_before_destroy = true + } + + network_interfaces { + delete_on_termination = true + associate_public_ip_address = true + subnet_id = var.subnet_id + security_groups = [ + aws_security_group.ec2_sg.id, + ] + } + + tag_specifications { + resource_type = "instance" + tags = var.tags + } +} + +resource "aws_instance" "ooni_ec2" { + launch_template { + id = aws_launch_template.ooni_ec2.id + version = "$Latest" + } + + lifecycle { + create_before_destroy = true + } + + tags = var.tags +} + +resource "aws_alb_target_group" "ooni_ec2" { + name_prefix = "oo${var.tg_prefix}" + port = 80 + protocol = "HTTP" + vpc_id = var.vpc_id + + lifecycle { + create_before_destroy = true + } + + tags = var.tags +} + +resource "aws_lb_target_group_attachment" "oonibackend_proxy" { + target_id = aws_instance.ooni_ec2.id + target_group_arn = aws_alb_target_group.ooni_ec2.arn +} diff --git a/tf/modules/ec2/outputs.tf b/tf/modules/ec2/outputs.tf new file mode 100644 index 00000000..4a99b3c8 --- /dev/null +++ b/tf/modules/ec2/outputs.tf @@ -0,0 +1,7 @@ +output "aws_instance_id" { + value = aws_instance.ooni_ec2.id +} + +output "aws_instance_public_dns" { + value = aws_instance.ooni_ec2.public_dns +} diff --git a/tf/modules/ec2/templates/cloud-init.yml b/tf/modules/ec2/templates/cloud-init.yml new file mode 100644 index 00000000..e69de29b diff --git a/tf/modules/ec2/variables.tf b/tf/modules/ec2/variables.tf new file mode 100644 index 00000000..78a02505 --- /dev/null +++ b/tf/modules/ec2/variables.tf @@ -0,0 +1,65 @@ +variable "vpc_id" { + description = "the id of the VPC to deploy the instance into" +} + +variable "subnet_id" { + description = "the ids of the subnet to deploy the instance into" +} + +variable "private_subnet_cidr" { + description = "the cidr block of the private subnet to allow traffic from for the clickhouse proxy" +} + + variable "tags" { + description = "tags to apply to the resources" + default = {} + type = map(string) +} + +variable "key_name" { + description = "Name of AWS key pair" +} + +variable "name" { + description = "Name of the resources" +} + +variable "instance_type" { + default = "t2.micro" +} + +variable "stage" { + default = "one of dev, stage, test, prod" +} + +variable "dns_zone_ooni_io" { + description = "id of the DNS zone for ooni_io" +} + +variable "sg_prefix" { + description = "security group prefix" +} + +variable "ingress_rules" { + type = list(object({ + from_port = number + to_port = number + protocol = string + cidr_blocks = list(string) + ipv6_cidr_blocks = optional(list(string)) + })) +} + +variable "egress_rules" { + type = list(object({ + from_port = number + to_port = number + protocol = string + cidr_blocks = optional(list(string)) + ipv6_cidr_blocks = optional(list(string)) + })) +} + +variable "tg_prefix" { + description = "target group prefix. Will be prefixed with `oo`, example: bkprx -> oobkprx" +} diff --git a/tf/modules/ooni_backendproxy/main.tf b/tf/modules/ooni_backendproxy/main.tf index 110461d3..a5674a60 100644 --- a/tf/modules/ooni_backendproxy/main.tf +++ b/tf/modules/ooni_backendproxy/main.tf @@ -14,7 +14,6 @@ resource "aws_security_group" "nginx_sg" { protocol = "tcp" from_port = 9000 to_port = 9000 - cidr_blocks = var.private_subnet_cidr } ingress { diff --git a/tf/modules/ooni_backendproxy/templates/cloud-init.yml b/tf/modules/ooni_backendproxy/templates/cloud-init.yml index 49663223..e69de29b 100644 --- a/tf/modules/ooni_backendproxy/templates/cloud-init.yml +++ b/tf/modules/ooni_backendproxy/templates/cloud-init.yml @@ -1,58 +0,0 @@ -package_update: true - -packages: - - nginx - - libnginx-mod-stream - -write_files: - - path: /etc/nginx/sites-available/default - content: | - server { - listen 80; - - server_name _; - - location / { - proxy_pass ${backend_url}; - proxy_http_version 1.1; - proxy_set_header Host \$host; - } - error_log /var/log/nginx/error.log; - } - - %{ if length(wcth_addresses) > 0 } - upstream wcths { - %{ for address in wcth_addresses } - server ${ address }; - %{ endfor } - } - server { - server_name *.${ wcth_domain_suffix }; - listen 80; - - location / { - proxy_pass http://wcths; - proxy_http_version 1.1; - proxy_set_header Host \$host; - } - } - %{ endif } - - - path: /etc/nginx/modules-enabled/99-stream.conf - content: | - stream { - upstream clickhouse_backend { - server ${clickhouse_url}:${clickhouse_port}; - } - - server { - listen 9000; - - proxy_pass clickhouse_backend; - } - - error_log /var/log/nginx/error.log; - } - -runcmd: - - service nginx restart From 281fb91f476e505c1aae22e926623b7be28a6f20 Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 9 Jan 2025 15:57:27 +0530 Subject: [PATCH 2/4] add clickhouse proxy role --- .../roles/clickhouse_proxy/handlers/main.yml | 15 ++++++++++++++ ansible/roles/clickhouse_proxy/tasks/main.yml | 20 +++++++++++++++++++ .../clickhouse_proxy/templates/99-stream.conf | 13 ++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 ansible/roles/clickhouse_proxy/handlers/main.yml create mode 100644 ansible/roles/clickhouse_proxy/tasks/main.yml create mode 100644 ansible/roles/clickhouse_proxy/templates/99-stream.conf diff --git a/ansible/roles/clickhouse_proxy/handlers/main.yml b/ansible/roles/clickhouse_proxy/handlers/main.yml new file mode 100644 index 00000000..eb1d1671 --- /dev/null +++ b/ansible/roles/clickhouse_proxy/handlers/main.yml @@ -0,0 +1,15 @@ +- name: test nginx config + command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf + listen: + - restart nginx + - reload nginx + +- name: restart nginx + service: + name: nginx + state: restarted + +- name: reload nginx + service: + name: nginx + state: reloaded diff --git a/ansible/roles/clickhouse_proxy/tasks/main.yml b/ansible/roles/clickhouse_proxy/tasks/main.yml new file mode 100644 index 00000000..5f18a27e --- /dev/null +++ b/ansible/roles/clickhouse_proxy/tasks/main.yml @@ -0,0 +1,20 @@ +--- +- name: Create the modules-enabled directory if not exists + tags: webserv + ansible.builtin.file: + path: /etc/nginx/modules-enabled + state: directory + mode: 0755 + owner: root + group: root + +- name: Add stream nginx config + tags: webserv + template: + src: templates/99-stream.conf + dest: /etc/nginx/modules-enabled/99-stream.conf + mode: 0755 + owner: root + notify: + - reload nginx + - restart nginx diff --git a/ansible/roles/clickhouse_proxy/templates/99-stream.conf b/ansible/roles/clickhouse_proxy/templates/99-stream.conf new file mode 100644 index 00000000..22f5a0f4 --- /dev/null +++ b/ansible/roles/clickhouse_proxy/templates/99-stream.conf @@ -0,0 +1,13 @@ +stream { + upstream clickhouse_backend { + server {{ clickhouse_url }}:{{ clickhouse_port }}; + } + + server { + listen 9000; + + proxy_pass clickhouse_backend; + } + + error_log /var/log/nginx/error.log; +} From 3a496c14431fc8ad647bd5ed9cd81534e2fa79c9 Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 9 Jan 2025 15:58:48 +0530 Subject: [PATCH 3/4] add clickhouse deployer notebook --- ansible/deploy-clickhouse-proxy.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 ansible/deploy-clickhouse-proxy.yml diff --git a/ansible/deploy-clickhouse-proxy.yml b/ansible/deploy-clickhouse-proxy.yml new file mode 100644 index 00000000..233ace4f --- /dev/null +++ b/ansible/deploy-clickhouse-proxy.yml @@ -0,0 +1,13 @@ +--- +- name: Deploy clickhouse proxy + hosts: + - clickhouseproxy.dev.ooni.io + become: true + roles: + - role: bootstrap + - role: nginx + tags: nginx + - role: clickhouse_proxy + vars: + clickhouse_url: "clickhouse3.prod.ooni.io" + clickhouse_port: 9000 From 919d05d9fc7db69a8f96fdcaf58303c3c1847d54 Mon Sep 17 00:00:00 2001 From: decfox Date: Thu, 9 Jan 2025 16:02:04 +0530 Subject: [PATCH 4/4] add clickhouse proxy to inventory --- ansible/inventory | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/inventory b/ansible/inventory index 6b24654a..0bb2870d 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -31,3 +31,6 @@ ams-ps.ooni.nu # currently disabled due to them not supporting ed25519 keys #mia-echoth.ooni.nu #mia-httpth.ooni.nu + +[aws-proxy] +clickhouseproxy.dev.ooni.io