[GCP] conformance terraform (#227)

deployment/live/gcp/cloudbuild/README.md

@@ -6,8 +6,8 @@ define the necessary triggers and steps in GCB.
 
 These steps will:
  1. Trigger on a change to the `main` branch of the trillian-tessera repo
- 2. Build the `example-gcp` docker image from the `main` branch
- 3. Publish this docker image in artifact repository
+ 2. Build the `cmd/gcp/conformance` docker image from the `main` branch
+ 3. Run conformance tests.
 
 The first time this is run for a pair of {GCP Project, GitHub Repo} you will get an error 
 message such as the following:

deployment/live/gcp/cloudbuild/prod/terragrunt.hcl

@@ -6,5 +6,7 @@ include "root" {
 inputs = merge(
   include.root.locals,
   {
+    kms_key_version_id = get_env("TESSERA_KMS_KEY_VERSION", "projects/${include.root.locals.project_id}/locations/${include.root.locals.region}/keyRings/ci-conformance/cryptoKeys/log-signer/cryptoKeyVersions/1")
+    log_origin         = "ci-conformance"
   }
 )

deployment/live/gcp/cloudbuild/terragrunt.hcl

@@ -1,5 +1,5 @@
 terraform {
-  source = "${get_repo_root()}/deployment/modules/gcp/cloudbuild"
+  source = "${get_repo_root()}/deployment/modules/gcp//cloudbuild"
 }
 
 locals {

deployment/live/gcp/conformance/README.md

@@ -1,14 +1,31 @@
-## Deployment 
+# GCP Conformance Configs
+
+## Prerequisites
+
+You'll need to have already configured/created a KMS key which can safely be used by the
+conformance log.
+
+> [Warning]
+> This key should not be used elsewhere or be in any way valuable!
+
+## Automatic deployment
+
+For the most part, this terragrunt config is automatically used as part conformance
+testing by the [CloudBuild](/deployment/live/cloudbuild) pipeline, so doesn't generally
+need to be manually applied.
+
+## Manual deployment 
 
 First authenticate via `gcloud` as a principle with sufficient ACLs for
 the project:
 ```bash
 gcloud auth application-default login
 ```
 
-Set your GCP project ID with:
+Set the required environment variables:
 ```bash
 export GOOGLE_PROJECT={VALUE}
+export TESSERA_KMS_KEY_VERSION={VALUE} # This should be the resource name of the key version created above
 ```
 
 Optionally, customize the GCP region (defaults to "us-central1"),

deployment/live/gcp/conformance/terragrunt.hcl

@@ -3,11 +3,15 @@ terraform {
 }
 
 locals {
-  env        = path_relative_to_include()
-  project_id = get_env("GOOGLE_PROJECT", "trillian-tessera")
-  location   = get_env("GOOGLE_REGION", "us-central1")
-  base_name  = get_env("TESSERA_BASE_NAME", "${local.env}-conformance")
-  log_origin = "conformance-gcp-${local.env}"
+  env                          = path_relative_to_include()
+  project_id                   = get_env("GOOGLE_PROJECT", "trillian-tessera")
+  location                     = get_env("GOOGLE_REGION", "us-central1")
+  base_name                    = get_env("TESSERA_BASE_NAME", "${local.env}-conformance")
+  conformance_gcp_docker_image = "${local.location}-docker.pkg.dev/trillian-tessera/docker-${local.env}/conformance-gcp:latest"
+  kms_key_version_id           = get_env("TESSERA_KMS_KEY_VERSION", "projects/${local.project_id}/locations/${local.location}/keyRings/${local.base_name}/cryptoKeys/log-signer/cryptoKeyVersions/1")
+  log_origin                   = local.base_name
+  conformance_users            = ["serviceAccount:[email protected]"]
+  bucket_readers               = ["serviceAccount:[email protected]"]
 }
 
 remote_state {

deployment/modules/gcp/cloudbuild/main.tf

@@ -1,32 +1,22 @@
 terraform {
-  required_providers {
-    google = {
-      source  = "hashicorp/google"
-      version = "5.41.0"
-    }
-  }
+  backend "gcs" {}
 }
 
 provider "google" {
   project = var.project_id
   region  = var.region
 }
 
-# This will be configured by terragrunt when deploying
-terraform {
-  backend "gcs" {}
-}
-
 resource "google_artifact_registry_repository" "docker" {
   repository_id = "docker-${var.env}"
   location      = var.region
-  description   = "Tessera example docker images"
+  description   = "Tessera conformance docker images"
   format        = "DOCKER"
 }
 
 locals {
-  artifact_repo            = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.docker.name}"
-  example_gcp_docker_image = "${local.artifact_repo}/example-gcp"
+  artifact_repo                = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.docker.name}"
+  conformance_gcp_docker_image = "${local.artifact_repo}/conformance-gcp"
 }
 
 resource "google_cloudbuild_trigger" "docker" {
@@ -43,55 +33,110 @@ resource "google_cloudbuild_trigger" "docker" {
   }
 
   build {
+    ## Build the GCP conformance server docker image.
+    ## This will be used by the conformance terragrunt config step further down.
     step {
+      id = "docker_build_conformance_gcp"
       name = "gcr.io/cloud-builders/docker"
       args = [
         "build",
-        "-t", "${local.example_gcp_docker_image}:$SHORT_SHA",
-        "-t", "${local.example_gcp_docker_image}:latest",
-        "-f", "./cmd/example-gcp/Dockerfile",
+        "-t", "${local.conformance_gcp_docker_image}:$SHORT_SHA",
+        "-t", "${local.conformance_gcp_docker_image}:latest",
+        "-f", "./cmd/conformance/gcp/Dockerfile",
         "."
       ]
     }
+    ## Push the image.
     step {
+      id   = "docker_push_conformance_gcp"
       name = "gcr.io/cloud-builders/docker"
       args = [
         "push",
         "--all-tags",
-        local.example_gcp_docker_image
+        local.conformance_gcp_docker_image
       ]
+      wait_for = ["docker_build_conformance_gcp"]
     }
+    ## Apply the deployment/live/gcp/conformance/ci terragrunt config.
+    ## This will bring up the conformance infrastructure, including a service
+    ## running the confirmance server docker image built above.
+    step {
+      id         = "terraform_apply_conformance_ci"
+      name       = "alpine/terragrunt"
+      entrypoint = "terragrunt"
+      args = [
+        "--terragrunt-non-interactive",
+        "apply",
+        "-auto-approve",
+      ]
+      dir = "deployment/live/gcp/conformance/ci"
+      env = [
+        "GOOGLE_PROJECT=${var.project_id}",
+        "TF_IN_AUTOMATION=1",
+        "TF_INPUT=false",
+        "TF_VAR_project_id=${var.project_id}"
+      ]
+      wait_for = ["docker_push_conformance_gcp"]
+    }
+    ## Grab some outputs from the terragrunt apply above (e.g. conformance server URL) and store
+    ## them in files under /workspace. These are needed for later steps.
+    step {
+      id = "terraform_outputs"
+      name = "alpine/terragrunt"
+      script = <<EOT
+        cd deployment/live/gcp/conformance/ci
+        terragrunt output --raw conformance_url > /workspace/conformance_url
+      EOT
+      wait_for = ["terraform_apply_conformance_ci"]
+    }
+    ## Build a note verifier string which can be used for verifying checkpoint signatures on the
+    ## conformange logs.
+    step {
+      id   = "generate_verifier"
+      name = "golang"
+      args = [
+        "go",
+        "run",
+        "./cmd/conformance/gcp/kmsnote",
+        "--key_id=${var.kms_key_version_id}",
+        "--name=${var.log_origin}",
+        "--output=/workspace/verifier.pub"
+      ]
+      wait_for = ["terraform_apply_conformance_ci"]
+    }
+    ## Since the conformance infrastructure is not publicly accessible, we need to use bearer tokens
+    ## for the hammer to access them.
+    ## This step creates those, and stores them for later use.
+    step { 
+      id = "access"
+      name = "gcr.io/cloud-builders/gcloud"
+      script = <<EOT
+      gcloud auth print-access-token > /workspace/cb_access
+      curl -H "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/service-accounts/${google_service_account.cloudbuild_service_account.email}/identity?audience=$(cat /workspace/conformance_url)" > /workspace/cb_identity
+      EOT
+      wait_for = ["terraform_outputs"]
+    }
+    ## Run the hammer against the conformance server.
+    ## We're using it in "target throughput" mode.
+    step {
+      id   = "hammer"
+      name = "golang"
+      script = <<EOT
+      go run ./hammer --log_public_key=$(cat /workspace/verifier.pub) --log_url=https://storage.googleapis.com/trillian-tessera-ci-conformance-bucket/ --write_log_url="$(cat /workspace/conformance_url)" -v=1 --show_ui=false --bearer_token="$(cat /workspace/cb_access)" --bearer_token_write="$(cat /workspace/cb_identity)" --logtostderr --num_writers=1100 --max_write_ops=1024 --leaf_min_size=1024 --leaf_write_goal=50000
+      EOT
+      wait_for = ["terraform_outputs", "generate_verifier", "access"]
+    }
+
     options {
       logging = "CLOUD_LOGGING_ONLY"
+      machine_type = "E2_HIGHCPU_8"
     }
   }
 }
 
+# roles managed externally.
 resource "google_service_account" "cloudbuild_service_account" {
   account_id   = "cloudbuild-${var.env}-sa"
   display_name = "Service Account for CloudBuild (${var.env})"
 }
 
-resource "google_project_iam_member" "act_as" {
-  project = var.project_id
-  role    = "roles/iam.serviceAccountUser"
-  member  = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
-}
-
-resource "google_project_iam_member" "logs_writer" {
-  project = var.project_id
-  role    = "roles/logging.logWriter"
-  member  = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
-}
-
-resource "google_project_iam_member" "artifact_registry_writer" {
-  project = var.project_id
-  role    = "roles/artifactregistry.writer"
-  member  = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
-}
-
-resource "google_project_iam_member" "cloudrun_deployer" {
-  project = var.project_id
-  role    = "roles/run.developer"
-  member  = "serviceAccount:${google_service_account.cloudbuild_service_account.email}"
-}

deployment/modules/gcp/cloudbuild/outputs.tf

@@ -15,5 +15,5 @@ output "cloudbuild_trigger_id" {
 
 output "docker_image" {
   description = "The address of the docker image that will be built"
-  value       = local.example_gcp_docker_image
+  value       = local.conformance_gcp_docker_image
 }

deployment/modules/gcp/cloudbuild/variables.tf

@@ -12,3 +12,15 @@ variable "env" {
   description = "Unique identifier for the env, e.g. ci or prod"
   type        = string
 }
+
+variable "log_origin" {
+  description = "The origin string for the conformance log"
+  type        = string
+}
+
+variable "kms_key_version_id" {
+  description = "The resource ID for the (externally created) KMS key version to use for signing checkpoints"
+  type        = string
+}
+
+