Merge pull request #58 from ooni/oonimeasurements

feat: oonimeasurements service deployment

.github/workflows/check_terraform.yml

@@ -41,7 +41,7 @@ jobs:
 
           [oonidevops_user_dev]
           aws_access_key_id = ${{ secrets.OONIDEVOPS_AWS_ACCESS_KEY_ID }}
-          aws_secret_access_key = ${{ secrets.OONIDEVOPS_AWS_SECRET_ACCESS_KEY }}
+          aws_secret_access_key = ${{ secrets.OONIDEVOPS_AWS_SECRET_ACCESS_KEY }} 
           EOF
           chmod 700 ~/.aws/
           chmod 600 ~/.aws/credentials
@@ -94,6 +94,7 @@ jobs:
           script: |
             const terraformPlanOutput = `${{ steps.plan.outputs.terraform_plan }}`;
             const terraformApplyOutput = `${{ steps.apply.outputs.terraform_apply }}`;
+            const terraformValidateOutput = `${{ steps.validate.outputs.terraform_validate }}`;
 
             const terraformPlanPlanLine = terraformPlanOutput.split('\n').find(line => line.startsWith('Plan:'));
             const terraformApplyPlanLine = terraformApplyOutput.split('\n').find(line => line.startsWith('Plan:'));
@@ -107,7 +108,7 @@ jobs:
             <details><summary>Validation Output</summary>
 
             \`\`\`\n
-            ${{ steps.validate.outputs.terraform_validate }}
+            ${terraformValidateOutput}
             \`\`\`
 
             </details>

tf/environments/dev/main.tf

@@ -489,7 +489,7 @@ module "ooniapi_oonifindings_deployer" {
 
   service_name            = "oonifindings"
   repo                    = "ooni/backend"
-  branch_name             = "oonidata"
+  branch_name             = "master"
   buildspec_path          = "ooniapi/services/oonifindings/buildspec.yml"
   codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn
 
@@ -597,6 +597,55 @@ module "ooniapi_ooniauth" {
   )
 }
 
+### OONI Measurements service
+
+module "ooniapi_oonimeasurements_deployer" {
+  source = "../../modules/ooniapi_service_deployer"
+
+  service_name            = "oonimeasurements"
+  repo                    = "ooni/backend"
+  branch_name             = "richer-analysis"
+  buildspec_path          = "ooniapi/services/oonimeasurements/buildspec.yml"
+  codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn
+
+  codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket
+
+  ecs_service_name = module.ooniapi_oonimeasurements.ecs_service_name
+  ecs_cluster_name = module.ooniapi_cluster.cluster_name
+}
+
+module "ooniapi_oonimeasurements" {
+  source = "../../modules/ooniapi_service"
+
+  task_memory = 64
+
+  first_run = true
+  vpc_id    = module.network.vpc_id
+
+  service_name             = "oonimeasurements"
+  default_docker_image_url = "ooni/api-oonimeasurements:latest"
+  stage                    = local.environment
+  dns_zone_ooni_io         = local.dns_zone_ooni_io
+  key_name                 = module.adm_iam_roles.oonidevops_key_name
+  ecs_cluster_id           = module.ooniapi_cluster.cluster_id
+
+  task_secrets = {
+    POSTGRESQL_URL              = aws_secretsmanager_secret_version.oonipg_url.arn
+    JWT_ENCRYPTION_KEY          = data.aws_ssm_parameter.jwt_secret.arn
+    PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn
+    CLICKHOUSE_URL              = data.aws_ssm_parameter.clickhouse_readonly_url.arn
+  }
+
+  ooniapi_service_security_groups = [
+    module.ooniapi_cluster.web_security_group_id
+  ]
+
+  tags = merge(
+    local.tags,
+    { Name = "ooni-tier0-oonimeasurements" }
+  )
+}
+
 #### OONI Tier0 API Frontend
 
 module "ooniapi_frontend" {
@@ -605,11 +654,12 @@ module "ooniapi_frontend" {
   vpc_id     = module.network.vpc_id
   subnet_ids = module.network.vpc_subnet_public[*].id
 
-  oonibackend_proxy_target_group_arn    = module.ooniapi_reverseproxy.alb_target_group_id
-  ooniapi_oonirun_target_group_arn      = module.ooniapi_oonirun.alb_target_group_id
-  ooniapi_ooniauth_target_group_arn     = module.ooniapi_ooniauth.alb_target_group_id
-  ooniapi_ooniprobe_target_group_arn    = module.ooniapi_ooniprobe.alb_target_group_id
-  ooniapi_oonifindings_target_group_arn = module.ooniapi_oonifindings.alb_target_group_id
+  oonibackend_proxy_target_group_arn        = module.ooniapi_reverseproxy.alb_target_group_id
+  ooniapi_oonirun_target_group_arn          = module.ooniapi_oonirun.alb_target_group_id
+  ooniapi_ooniauth_target_group_arn         = module.ooniapi_ooniauth.alb_target_group_id
+  ooniapi_ooniprobe_target_group_arn        = module.ooniapi_ooniprobe.alb_target_group_id
+  ooniapi_oonifindings_target_group_arn     = module.ooniapi_oonifindings.alb_target_group_id
+  ooniapi_oonimeasurements_target_group_arn = module.ooniapi_oonimeasurements.alb_target_group_id
 
   ooniapi_service_security_groups = [
     module.ooniapi_cluster.web_security_group_id

tf/modules/ooniapi_frontend/main.tf

@@ -184,9 +184,6 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" {
     path_pattern {
       values = [
         "/api/v1/incidents/*",
-        "/api/v1/aggregation/*",
-        "/api/v1/observations",
-        "/api/v1/analysis",
       ]
     }
   }
@@ -205,4 +202,71 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule_host" {
       values = ["oonifindings.${local.direct_domain_suffix}"]
     }
   }
-}
+}
+
+resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_1" {
+  # hotfix: to allow us to deploy the frontend without the measurements service
+  count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0
+
+  listener_arn = aws_alb_listener.ooniapi_listener_https.arn
+  priority     = 140
+
+  action {
+    type             = "forward"
+    target_group_arn = var.ooniapi_oonimeasurements_target_group_arn
+  }
+
+  condition {
+    path_pattern {
+      values = [
+        "/api/v1/measurements/*",
+        "/api/v1/raw_measurement",
+        "/api/v1/measurement_meta",
+        "/api/v1/measurements",
+        "/api/v1/torsf_stats"
+      ]
+    }
+  }
+}
+
+resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_2" {
+  # hotfix: to allow us to deploy the frontend without the measurements service
+  count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0
+
+  listener_arn = aws_alb_listener.ooniapi_listener_https.arn
+  priority     = 142
+
+  action {
+    type             = "forward"
+    target_group_arn = var.ooniapi_oonimeasurements_target_group_arn
+  }
+
+  condition {
+    path_pattern {
+      values = [
+        "/api/v1/aggregation",
+        "/api/v1/aggregation/*",
+        "/api/v1/observations",
+        "/api/v1/analysis",
+      ]
+    }
+  }
+}
+
+resource "aws_lb_listener_rule" "ooniapi_oonimeasurements_rule_host" {
+  # hotfix: to allow us to deploy the frontend without the measurements service
+  count = var.ooniapi_oonimeasurements_target_group_arn != null ? 1 : 0
+
+  listener_arn = aws_alb_listener.ooniapi_listener_https.arn
+  priority     = 141
+
+  action {
+    type             = "forward"
+    target_group_arn = var.ooniapi_oonimeasurements_target_group_arn
+  }
+  condition {
+    host_header {
+      values = ["oonimeasurements.${local.direct_domain_suffix}"]
+    }
+  }
+}

tf/modules/ooniapi_frontend/variables.tf

@@ -32,6 +32,11 @@ variable "ooniapi_oonifindings_target_group_arn" {
   description = "arn for the target group of the oonifindings service"
 }
 
+variable "ooniapi_oonimeasurements_target_group_arn" {
+  description = "arn for the target group of the oonimeasurements service"
+  default     = null
+}
+
 variable "dns_zone_ooni_io" {
   description = "id of the DNS zone for ooni_io"
 }
@@ -52,4 +57,4 @@ variable "oonith_domains" {
 
 variable "ooniapi_acm_certificate_arn" {
   type = string
-}
+}