From c65f27721999262bbd9755df3738171d43d63869 Mon Sep 17 00:00:00 2001
From: Al Cutter <al@google.com>
Date: Wed, 11 Sep 2024 18:16:44 +0100
Subject: [PATCH] [GCP] conformance terraform  (#227)

---
 deployment/live/gcp/cloudbuild/README.md      |   4 +-
 .../live/gcp/cloudbuild/prod/terragrunt.hcl   |   2 +
 deployment/live/gcp/cloudbuild/terragrunt.hcl |   2 +-
 deployment/live/gcp/conformance/README.md     |  21 ++-
 .../live/gcp/conformance/terragrunt.hcl       |  14 +-
 deployment/modules/gcp/cloudbuild/main.tf     | 127 ++++++++++++------
 deployment/modules/gcp/cloudbuild/outputs.tf  |   2 +-
 .../modules/gcp/cloudbuild/variables.tf       |  12 ++
 deployment/modules/gcp/conformance/main.tf    |  90 ++++++++-----
 deployment/modules/gcp/conformance/outputs.tf |   9 ++
 .../modules/gcp/conformance/variables.tf      |  15 +++
 deployment/modules/gcp/gcs/main.tf            |   9 ++
 deployment/modules/gcp/gcs/variables.tf       |   6 +
 13 files changed, 228 insertions(+), 85 deletions(-)
 create mode 100644 deployment/modules/gcp/conformance/outputs.tf

diff --git a/deployment/live/gcp/cloudbuild/README.md b/deployment/live/gcp/cloudbuild/README.md
index bee77c57..d040fa7e 100644
--- a/deployment/live/gcp/cloudbuild/README.md
+++ b/deployment/live/gcp/cloudbuild/README.md
@@ -6,8 +6,8 @@ define the necessary triggers and steps in GCB.
 
 These steps will:
  1. Trigger on a change to the `main` branch of the trillian-tessera repo
- 2. Build the `example-gcp` docker image from the `main` branch
- 3. Publish this docker image in artifact repository
+ 2. Build the `cmd/gcp/conformance` docker image from the `main` branch
+ 3. Run conformance tests.
 
 The first time this is run for a pair of {GCP Project, GitHub Repo} you will get an error 
 message such as the following:
diff --git a/deployment/live/gcp/cloudbuild/prod/terragrunt.hcl b/deployment/live/gcp/cloudbuild/prod/terragrunt.hcl
index 22da2c1b..bfa6b083 100644
--- a/deployment/live/gcp/cloudbuild/prod/terragrunt.hcl
+++ b/deployment/live/gcp/cloudbuild/prod/terragrunt.hcl
@@ -6,5 +6,7 @@ include "root" {
 inputs = merge(
   include.root.locals,
   {
+    kms_key_version_id = get_env("TESSERA_KMS_KEY_VERSION", "projects/${include.root.locals.project_id}/locations/${include.root.locals.region}/keyRings/ci-conformance/cryptoKeys/log-signer/cryptoKeyVersions/1")
+    log_origin         = "ci-conformance"
   }
 )
diff --git a/deployment/live/gcp/cloudbuild/terragrunt.hcl b/deployment/live/gcp/cloudbuild/terragrunt.hcl
index d1bb698b..4ebf2f97 100644
--- a/deployment/live/gcp/cloudbuild/terragrunt.hcl
+++ b/deployment/live/gcp/cloudbuild/terragrunt.hcl
@@ -1,5 +1,5 @@
 terraform {
-  source = "${get_repo_root()}/deployment/modules/gcp/cloudbuild"
+  source = "${get_repo_root()}/deployment/modules/gcp//cloudbuild"
 }
 
 locals {
diff --git a/deployment/live/gcp/conformance/README.md b/deployment/live/gcp/conformance/README.md
index c5c5a0f5..21bba59d 100644
--- a/deployment/live/gcp/conformance/README.md
+++ b/deployment/live/gcp/conformance/README.md
@@ -1,4 +1,20 @@
-## Deployment 
+# GCP Conformance Configs
+
+## Prerequisites
+
+You'll need to have already configured/created a KMS key which can safely be used by the
+conformance log.
+
+> [Warning]
+> This key should not be used elsewhere or be in any way valuable!
+
+## Automatic deployment
+
+For the most part, this terragrunt config is automatically used as part conformance
+testing by the [CloudBuild](/deployment/live/cloudbuild) pipeline, so doesn't generally
+need to be manually applied.
+
+## Manual deployment 
 
 First authenticate via `gcloud` as a principle with sufficient ACLs for
 the project:
@@ -6,9 +22,10 @@ the project:
 gcloud auth application-default login
 ```
 
-Set your GCP project ID with:
+Set the required environment variables:
 ```bash
 export GOOGLE_PROJECT={VALUE}
+export TESSERA_KMS_KEY_VERSION={VALUE} # This should be the resource name of the key version created above
 ```
 
 Optionally, customize the GCP region (defaults to "us-central1"),
diff --git a/deployment/live/gcp/conformance/terragrunt.hcl b/deployment/live/gcp/conformance/terragrunt.hcl
index bb0b4da7..1c77b85a 100644
--- a/deployment/live/gcp/conformance/terragrunt.hcl
+++ b/deployment/live/gcp/conformance/terragrunt.hcl
@@ -3,11 +3,15 @@ terraform {
 }
 
 locals {
-  env        = path_relative_to_include()
-  project_id = get_env("GOOGLE_PROJECT", "trillian-tessera")
-  location   = get_env("GOOGLE_REGION", "us-central1")
-  base_name  = get_env("TESSERA_BASE_NAME", "${local.env}-conformance")
-  log_origin = "conformance-gcp-${local.env}"
+  env                          = path_relative_to_include()
+  project_id                   = get_env("GOOGLE_PROJECT", "trillian-tessera")
+  location                     = get_env("GOOGLE_REGION", "us-central1")
+  base_name                    = get_env("TESSERA_BASE_NAME", "${local.env}-conformance")
+  conformance_gcp_docker_image = "${local.location}-docker.pkg.dev/trillian-tessera/docker-${local.env}/conformance-gcp:latest"
+  kms_key_version_id           = get_env("TESSERA_KMS_KEY_VERSION", "projects/${local.project_id}/locations/${local.location}/keyRings/${local.base_name}/cryptoKeys/log-signer/cryptoKeyVersions/1")
+  log_origin                   = local.base_name
+  conformance_users            = ["serviceAccount:cloudbuild-prod-sa@trillian-tessera.iam.gserviceaccount.com"]
+  bucket_readers               = ["serviceAccount:cloudbuild-prod-sa@trillian-tessera.iam.gserviceaccount.com"]
 }
 
 remote_state {
diff --git a/deployment/modules/gcp/cloudbuild/main.tf b/deployment/modules/gcp/cloudbuild/main.tf
index 52674ee5..ec33e443 100644
--- a/deployment/modules/gcp/cloudbuild/main.tf
+++ b/deployment/modules/gcp/cloudbuild/main.tf
@@ -1,10 +1,5 @@
 terraform {
-  required_providers {
-    google = {
-      source  = "hashicorp/google"
-      version = "5.41.0"
-    }
-  }
+  backend "gcs" {}
 }
 
 provider "google" {
@@ -12,21 +7,16 @@ provider "google" {
   region  = var.region
 }
 
-# This will be configured by terragrunt when deploying
-terraform {
-  backend "gcs" {}
-}
-
 resource "google_artifact_registry_repository" "docker" {
   repository_id = "docker-${var.env}"
   location      = var.region
-  description   = "Tessera example docker images"
+  description   = "Tessera conformance docker images"
   format        = "DOCKER"
 }
 
 locals {
-  artifact_repo            = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.docker.name}"
-  example_gcp_docker_image = "${local.artifact_repo}/example-gcp"
+  artifact_repo                = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.docker.name}"
+  conformance_gcp_docker_image = "${local.artifact_repo}/conformance-gcp"
 }
 
 resource "google_cloudbuild_trigger" "docker" {
@@ -43,55 +33,110 @@ resource "google_cloudbuild_trigger" "docker" {
   }
 
   build {
+    ## Build the GCP conformance server docker image.
+    ## This will be used by the conformance terragrunt config step further down.
     step {
+      id = "docker_build_conformance_gcp"
       name = "gcr.io/cloud-builders/docker"
       args = [
         "build",
-        "-t", "${local.example_gcp_docker_image}:$SHORT_SHA",
-        "-t", "${local.example_gcp_docker_image}:latest",
-        "-f", "./cmd/example-gcp/Dockerfile",
+        "-t", "${local.conformance_gcp_docker_image}:$SHORT_SHA",
+        "-t", "${local.conformance_gcp_docker_image}:latest",
+        "-f", "./cmd/conformance/gcp/Dockerfile",
         "."
       ]
     }
+    ## Push the image.
     step {
+      id   = "docker_push_conformance_gcp"
       name = "gcr.io/cloud-builders/docker"
       args = [
         "push",
         "--all-tags",
-        local.example_gcp_docker_image
+        local.conformance_gcp_docker_image
       ]
+      wait_for = ["docker_build_conformance_gcp"]
     }
+    ## Apply the deployment/live/gcp/conformance/ci terragrunt config.
+    ## This will bring up the conformance infrastructure, including a service
+    ## running the confirmance server docker image built above.
+    step {
+      id         = "terraform_apply_conformance_ci"
+      name       = "alpine/terragrunt"
+      entrypoint = "terragrunt"
+      args = [
+        "--terragrunt-non-interactive",
+        "apply",
+        "-auto-approve",
+      ]
+      dir = "deployment/live/gcp/conformance/ci"
+      env = [
+        "GOOGLE_PROJECT=${var.project_id}",
+        "TF_IN_AUTOMATION=1",
+        "TF_INPUT=false",
+        "TF_VAR_project_id=${var.project_id}"
+      ]
+      wait_for = ["docker_push_conformance_gcp"]
+    }
+    ## Grab some outputs from the terragrunt apply above (e.g. conformance server URL) and store
+    ## them in files under /workspace. These are needed for later steps.
+    step {
+      id = "terraform_outputs"
+      name = "alpine/terragrunt"
+      script = <<EOT
+        cd deployment/live/gcp/conformance/ci
+        terragrunt output --raw conformance_url > /workspace/conformance_url
+      EOT
+      wait_for = ["terraform_apply_conformance_ci"]
+    }
+    ## Build a note verifier string which can be used for verifying checkpoint signatures on the
+    ## conformange logs.
+    step {
+      id   = "generate_verifier"
+      name = "golang"
+      args = [
+        "go",
+        "run",
+        "./cmd/conformance/gcp/kmsnote",
+        "--key_id=${var.kms_key_version_id}",
+        "--name=${var.log_origin}",
+        "--output=/workspace/verifier.pub"
+      ]
+      wait_for = ["terraform_apply_conformance_ci"]
+    }
+    ## Since the conformance infrastructure is not publicly accessible, we need to use bearer tokens
+    ## for the hammer to access them.
+    ## This step creates those, and stores them for later use.
+    step { 
+      id = "access"
+      name = "gcr.io/cloud-builders/gcloud"
+      script = <<EOT
+      gcloud auth print-access-token > /workspace/cb_access
+      curl -H "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/service-accounts/${google_service_account.cloudbuild_service_account.email}/identity?audience=$(cat /workspace/conformance_url)" > /workspace/cb_identity
+      EOT
+      wait_for = ["terraform_outputs"]
+    }
+    ## Run the hammer against the conformance server.
+    ## We're using it in "target throughput" mode.
+    step {
+      id   = "hammer"
+      name = "golang"
+      script = <<EOT
+      go run ./hammer --log_public_key=$(cat /workspace/verifier.pub) --log_url=https://storage.googleapis.com/trillian-tessera-ci-conformance-bucket/ --write_log_url="$(cat /workspace/conformance_url)" -v=1 --show_ui=false --bearer_token="$(cat /workspace/cb_access)" --bearer_token_write="$(cat /workspace/cb_identity)" --logtostderr --num_writers=1100 --max_write_ops=1024 --leaf_min_size=1024 --leaf_write_goal=50000
+      EOT
+      wait_for = ["terraform_outputs", "generate_verifier", "access"]
+    }
+
     options {
       logging = "CLOUD_LOGGING_ONLY"
+      machine_type = "E2_HIGHCPU_8"
     }
   }
 }
 
+# roles managed externally.
 resource "google_service_account" "cloudbuild_service_account" {
   account_id   = "cloudbuild-${var.env}-sa"
   display_name = "Service Account for CloudBuild (${var.env})"
 }
 
-resource "google_project_iam_member" "act_as" {
-  project = var.project_id
-  role    = "roles/iam.serviceAccountUser"
-  member  = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
-}
-
-resource "google_project_iam_member" "logs_writer" {
-  project = var.project_id
-  role    = "roles/logging.logWriter"
-  member  = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
-}
-
-resource "google_project_iam_member" "artifact_registry_writer" {
-  project = var.project_id
-  role    = "roles/artifactregistry.writer"
-  member  = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
-}
-
-resource "google_project_iam_member" "cloudrun_deployer" {
-  project = var.project_id
-  role    = "roles/run.developer"
-  member  = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
-}
diff --git a/deployment/modules/gcp/cloudbuild/outputs.tf b/deployment/modules/gcp/cloudbuild/outputs.tf
index 99fc8d20..72776d22 100644
--- a/deployment/modules/gcp/cloudbuild/outputs.tf
+++ b/deployment/modules/gcp/cloudbuild/outputs.tf
@@ -15,5 +15,5 @@ output "cloudbuild_trigger_id" {
 
 output "docker_image" {
   description = "The address of the docker image that will be built"
-  value       = local.example_gcp_docker_image
+  value       = local.conformance_gcp_docker_image
 }
diff --git a/deployment/modules/gcp/cloudbuild/variables.tf b/deployment/modules/gcp/cloudbuild/variables.tf
index 4ca35a0d..b6966586 100644
--- a/deployment/modules/gcp/cloudbuild/variables.tf
+++ b/deployment/modules/gcp/cloudbuild/variables.tf
@@ -12,3 +12,15 @@ variable "env" {
   description = "Unique identifier for the env, e.g. ci or prod"
   type        = string
 }
+
+variable "log_origin" {
+  description = "The origin string for the conformance log"
+  type        = string
+}
+
+variable "kms_key_version_id" {
+  description = "The resource ID for the (externally created) KMS key version to use for signing checkpoints"
+  type        = string
+}
+
+
diff --git a/deployment/modules/gcp/conformance/main.tf b/deployment/modules/gcp/conformance/main.tf
index eb8164b4..6a7c4320 100644
--- a/deployment/modules/gcp/conformance/main.tf
+++ b/deployment/modules/gcp/conformance/main.tf
@@ -1,5 +1,12 @@
 terraform {
   backend "gcs" {}
+
+  required_providers {
+    google = {
+      source  = "registry.terraform.io/hashicorp/google"
+      version = "6.1.0"
+    }
+  }
 }
 
 ## Call the Tessera GCP module
@@ -12,6 +19,7 @@ module "gcs" {
   env        = var.env
   location   = var.location
   project_id = var.project_id
+  bucket_readers = var.bucket_readers
 }
 
 ##
@@ -27,60 +35,55 @@ resource "google_project_service" "cloudkms_googleapis_com" {
   service = "cloudkms.googleapis.com"
 }
 
+/*
+## This KMS config is left here for reference, but commented out to avoid
+## attempts to delete and re-create these keys with each of the conformance
+## runs.
+
 ##
 ## KMS for log signing
 ##
 resource "google_kms_key_ring" "log_signer" {
   location = var.location
   name     = var.base_name
+
+  lifecycle {
+    prevent_destroy = true
+  }
 }
 
 resource "google_kms_crypto_key" "log_signer" {
-  key_ring = google_kms_key_ring.log_signer.id
-  name     = "log_signer"
+  key_ring = google_kms_key_ring.log-signer.id
+  name     = "log-signer"
   purpose  = "ASYMMETRIC_SIGN"
   version_template {
     algorithm = "EC_SIGN_ED25519"
   }
+
+  lifecycle {
+    prevent_destroy = true
+  }
 }
+
 resource "google_kms_crypto_key_version" "log_signer" {
   crypto_key = google_kms_crypto_key.log_signer.id
+
+  lifecycle {
+    prevent_destroy = true
+  }
 }
+*/
+
 
 ###
 ### Set up Cloud Run service
 ###
+### Roles managed externally.
 resource "google_service_account" "cloudrun_service_account" {
   account_id   = "cloudrun-${var.env}-sa"
   display_name = "Service Account for Cloud Run (${var.base_name})"
 }
 
-resource "google_project_iam_member" "iam_act_as" {
-  project = var.project_id
-  role    = "roles/iam.serviceAccountUser"
-  member  = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
-}
-resource "google_project_iam_member" "iam_metrics_writer" {
-  project = var.project_id
-  role    = "roles/monitoring.metricWriter"
-  member  = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
-}
-resource "google_spanner_database_iam_binding" "iam_spanner_database_user" {
-  project  = var.project_id
-  instance = module.gcs.log_spanner_instance.name
-  database = module.gcs.log_spanner_db.name
-  role     = "roles/spanner.databaseUser"
-
-  members = [
-    "serviceAccount:${google_service_account.cloudrun_service_account.email}"
-  ]
-}
-resource "google_project_iam_member" "iam_service_agent" {
-  project = var.project_id
-  role    = "roles/run.serviceAgent"
-  member  = "serviceAccount:${google_service_account.cloudrun_service_account.email}"
-}
-
 locals {
   spanner_db_full = "projects/${var.project_id}/instances/${module.gcs.log_spanner_instance.name}/databases/${module.gcs.log_spanner_db.name}"
 }
@@ -92,6 +95,13 @@ resource "google_cloud_run_v2_service" "default" {
 
   template {
     service_account = google_service_account.cloudrun_service_account.email
+    max_instance_request_concurrency = 700
+    timeout = "10s"
+
+    scaling {
+      max_instance_count = 4
+    }
+
     containers {
       image = var.server_docker_image
       name  = "conformance"
@@ -102,13 +112,20 @@ resource "google_cloud_run_v2_service" "default" {
         "--spanner=${local.spanner_db_full}",
         "--project=${var.project_id}",
         "--listen=:8080",
-        "--kms_key=${google_kms_crypto_key_version.log_signer.id}",
+        "--kms_key=${var.kms_key_version_id}",
         "--origin=${var.log_origin}",
       ]
       ports {
         container_port = 8080
       }
 
+      resources {
+        limits = {
+          cpu = "2"
+          memory = "512Mi"
+        }
+      }
+
       startup_probe {
         initial_delay_seconds = 1
         timeout_seconds       = 1
@@ -120,14 +137,21 @@ resource "google_cloud_run_v2_service" "default" {
       }
     }
   }
+
+  deletion_protection = false
+
   client = "terraform"
   depends_on = [
     module.gcs,
     google_project_service.cloudrun_api,
-    google_project_iam_member.iam_act_as,
-    google_project_iam_member.iam_metrics_writer,
-    google_project_iam_member.iam_service_agent,
-    google_spanner_database_iam_binding.iam_spanner_database_user,
   ]
 }
 
+resource "google_cloud_run_v2_service_iam_binding" "cloudrun_invoker" {
+  location = google_cloud_run_v2_service.default.location
+  name     = google_cloud_run_v2_service.default.name
+  role     = "roles/run.invoker"
+  members  = var.conformance_users
+}
+
+
diff --git a/deployment/modules/gcp/conformance/outputs.tf b/deployment/modules/gcp/conformance/outputs.tf
new file mode 100644
index 00000000..413a3d62
--- /dev/null
+++ b/deployment/modules/gcp/conformance/outputs.tf
@@ -0,0 +1,9 @@
+output "run_service_account" {
+  description = "The CloudRun service account"
+  value       = google_service_account.cloudrun_service_account
+}
+
+output "conformance_url" {
+  description = "The URL of the running conformance server"
+  value       = google_cloud_run_v2_service.default.uri
+}
diff --git a/deployment/modules/gcp/conformance/variables.tf b/deployment/modules/gcp/conformance/variables.tf
index ffa823d6..84d81bac 100644
--- a/deployment/modules/gcp/conformance/variables.tf
+++ b/deployment/modules/gcp/conformance/variables.tf
@@ -27,3 +27,18 @@ variable "log_origin" {
   description = "The origin string for the conformance log"
   type        = string
 }
+
+variable "kms_key_version_id" {
+  description = "The resource ID for the (externally created) KMS key version to use for signing checkpoints"
+  type        = string
+}
+
+variable "conformance_users" {
+  description = "The list of users allowed to invoke calls to the conformance instance."
+  type = list
+}
+
+variable "bucket_readers" {
+  description = "The list of users allowed to read the conformance bucket contents"
+  type = list
+}
diff --git a/deployment/modules/gcp/gcs/main.tf b/deployment/modules/gcp/gcs/main.tf
index 8b3042ad..b47c3cfe 100644
--- a/deployment/modules/gcp/gcs/main.tf
+++ b/deployment/modules/gcp/gcs/main.tf
@@ -35,6 +35,15 @@ resource "google_storage_bucket" "log_bucket" {
   uniform_bucket_level_access = true
 }
 
+resource "google_storage_bucket_iam_binding" "log_bucket_reader" {
+  bucket = google_storage_bucket.log_bucket.name
+  role   = "roles/storage.objectViewer"
+  members = concat(
+    [ google_service_account.log_writer.member ],
+    var.bucket_readers
+  )
+}
+
 resource "google_storage_bucket_iam_binding" "log_bucket_writer" {
   bucket = google_storage_bucket.log_bucket.name
   role   = "roles/storage.legacyBucketWriter"
diff --git a/deployment/modules/gcp/gcs/variables.tf b/deployment/modules/gcp/gcs/variables.tf
index 71c3d46d..3d852722 100644
--- a/deployment/modules/gcp/gcs/variables.tf
+++ b/deployment/modules/gcp/gcs/variables.tf
@@ -17,3 +17,9 @@ variable "env" {
   description = "Unique identifier for the env, e.g. ci or prod"
   type        = string
 }
+
+variable "bucket_readers" {
+  description = "List of identities allowed to read the log bucket"
+  type = list
+  default = [ "allUsers" ]
+}