From 89d2c491fb37fcf564c3a3144cfc33f07ce0e607 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Fri, 20 Sep 2024 11:58:18 +0000 Subject: [PATCH 1/4] Add deployment configs and instructions --- deployment/live/gcp/test/README.md | 43 ++++++++++++++ deployment/live/gcp/test/terragrunt.hcl | 26 ++++++++ deployment/modules/gcp/storage/main.tf | 66 +++++++++++++++++++++ deployment/modules/gcp/storage/outputs.tf | 19 ++++++ deployment/modules/gcp/storage/variables.tf | 14 +++++ 5 files changed, 168 insertions(+) create mode 100644 deployment/live/gcp/test/README.md create mode 100644 deployment/live/gcp/test/terragrunt.hcl create mode 100644 deployment/modules/gcp/storage/main.tf create mode 100644 deployment/modules/gcp/storage/outputs.tf create mode 100644 deployment/modules/gcp/storage/variables.tf diff --git a/deployment/live/gcp/test/README.md b/deployment/live/gcp/test/README.md new file mode 100644 index 0000000..ed3b234 --- /dev/null +++ b/deployment/live/gcp/test/README.md @@ -0,0 +1,43 @@ +# GCP SCTFE Configs + +## Prerequisites +You'll need to have a VM running in the same GCP project that you can SSH to, +with go installed. + +## Overview + +This config uses the [gcp/storage](/deployment/modules/gcp/conformance) module to +define a test environment to run the SCTFE, backed by Trillian Tessera. + +At a high level, this environment consists of: +- Spanner DB +- GCS Bucket +- VM to run the code + +## Manual deployment + +This + +First authenticate via `gcloud` as a principle with sufficient ACLs for +the project: +```bash +gcloud auth application-default login +``` + +Set the required environment variables: +```bash +export GOOGLE_PROJECT={VALUE} +export GOOGLE_REGION={VALUE} # e.g: us-central1 +export TESSERA_BASE_NAME={VALUE} # e.g: staticct +``` + +Terraforming the project can be done by: + 1. `cd` to the relevant directory for the environment to deploy/change (e.g. `ci`) + 2. Run `terragrunt apply` + +## Run the SCTFE + +Run the following command: +```bash +go run ./cmd/gcp/ --project_id=${GOOGLE_PROJECT} --bucket=${GOOGLE_PROJECT}-${TESSERA_BASE_NAME}-bucket --spanner_db_path=projects/${GOOGLE_PROJECT}/instances/${TESSERA_BASE_NAME}/databases/${TESSERA_BASE_NAME}-db --spanner_db_path=projects/${GOOGLE_PROJECT}/instances/${TESSERA_BASE_NAME}/databases/${TESSERA_BASE_NAME}-dedup-db --private_key=./testdata/ct-http-server.privkey.pem --password=dirk --roots_pem_file=./testdata/fake-ca.cert --origin=${TESSERA_BASE_NAME} +``` diff --git a/deployment/live/gcp/test/terragrunt.hcl b/deployment/live/gcp/test/terragrunt.hcl new file mode 100644 index 0000000..2431f8a --- /dev/null +++ b/deployment/live/gcp/test/terragrunt.hcl @@ -0,0 +1,26 @@ +terraform { + source = "${get_repo_root()}/deployment/modules/gcp//storage" +} + +locals { + project_id = get_env("GOOGLE_PROJECT", "phboneff-dev") + location = get_env("GOOGLE_REGION", "us-central1") + base_name = get_env("TESSERA_BASE_NAME", "tessera-staticct") +} + +inputs = local + +remote_state { + backend = "gcs" + + config = { + project = local.project_id + location = local.location + bucket = "${local.project_id}-${local.base_name}-terraform-state" + prefix = "terraform.tfstate" + + gcs_bucket_labels = { + name = "terraform_state_storage" + } + } +} diff --git a/deployment/modules/gcp/storage/main.tf b/deployment/modules/gcp/storage/main.tf new file mode 100644 index 0000000..e3b6bc7 --- /dev/null +++ b/deployment/modules/gcp/storage/main.tf @@ -0,0 +1,66 @@ +terraform { + backend "gcs" {} + + required_providers { + google = { + source = "registry.terraform.io/hashicorp/google" + version = "6.1.0" + } + } +} + +# Services +resource "google_project_service" "serviceusage_googleapis_com" { + service = "serviceusage.googleapis.com" + disable_on_destroy = false +} +resource "google_project_service" "storage_api_googleapis_com" { + service = "storage-api.googleapis.com" + disable_on_destroy = false +} +resource "google_project_service" "storage_component_googleapis_com" { + service = "storage-component.googleapis.com" + disable_on_destroy = false +} +resource "google_project_service" "storage_googleapis_com" { + service = "storage.googleapis.com" + disable_on_destroy = false +} + +## Resources + +# Buckets + +resource "google_storage_bucket" "log_bucket" { + name = "${var.project_id}-${var.base_name}-bucket" + location = var.location + storage_class = "STANDARD" + uniform_bucket_level_access = true +} + +# Spanner + +resource "google_spanner_instance" "log_spanner" { + name = var.base_name + config = "regional-${var.location}" + display_name = var.base_name + processing_units = 100 +} + +resource "google_spanner_database" "log_db" { + instance = google_spanner_instance.log_spanner.name + name = "${var.base_name}-db" + ddl = [ + "CREATE TABLE SeqCoord (id INT64 NOT NULL, next INT64 NOT NULL,) PRIMARY KEY (id)", + "CREATE TABLE Seq (id INT64 NOT NULL, seq INT64 NOT NULL, v BYTES(MAX),) PRIMARY KEY (id, seq)", + "CREATE TABLE IntCoord (id INT64 NOT NULL, seq INT64 NOT NULL,) PRIMARY KEY (id)", + ] +} + +resource "google_spanner_database" "dedup_db" { + instance = google_spanner_instance.log_spanner.name + name = "${var.base_name}-dedup-db" + ddl = [ + "CREATE TABLE IDSeq (id INT64 NOT NULL, h BYTES(MAX) NOT NULL, idx INT64 NOT NULL,) PRIMARY KEY (id, h)", + ] +} diff --git a/deployment/modules/gcp/storage/outputs.tf b/deployment/modules/gcp/storage/outputs.tf new file mode 100644 index 0000000..c223976 --- /dev/null +++ b/deployment/modules/gcp/storage/outputs.tf @@ -0,0 +1,19 @@ +output "log_bucket" { + description = "Log GCS bucket" + value = google_storage_bucket.log_bucket +} + +output "log_spanner_db" { + description = "Log Spanner database" + value = google_spanner_database.log_db +} + +output "log_spanner_instance" { + description = "Log Spanner instance" + value = google_spanner_instance.log_spanner +} + +output "dedup_spanner_db" { + description = "Dedup Spanner database" + value = google_spanner_database.dedup_db +} diff --git a/deployment/modules/gcp/storage/variables.tf b/deployment/modules/gcp/storage/variables.tf new file mode 100644 index 0000000..fa8142e --- /dev/null +++ b/deployment/modules/gcp/storage/variables.tf @@ -0,0 +1,14 @@ +variable "project_id" { + description = "GCP project ID where the log is hosted" + type = string +} + +variable "base_name" { + description = "Base name to use when naming resources" + type = string +} + +variable "location" { + description = "Location in which to create resources" + type = string +} From 899485e808f526fac82b2ff3bc41254c33f33f49 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Tue, 8 Oct 2024 08:51:43 +0000 Subject: [PATCH 2/4] few fixes --- deployment/live/gcp/test/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/deployment/live/gcp/test/README.md b/deployment/live/gcp/test/README.md index ed3b234..5b8e6a0 100644 --- a/deployment/live/gcp/test/README.md +++ b/deployment/live/gcp/test/README.md @@ -10,14 +10,13 @@ This config uses the [gcp/storage](/deployment/modules/gcp/conformance) module t define a test environment to run the SCTFE, backed by Trillian Tessera. At a high level, this environment consists of: -- Spanner DB -- GCS Bucket -- VM to run the code +- One Spanner instance with two databases: + - one for Tessera + - one for deduplication +- A GCS Bucket ## Manual deployment -This - First authenticate via `gcloud` as a principle with sufficient ACLs for the project: ```bash @@ -37,7 +36,7 @@ Terraforming the project can be done by: ## Run the SCTFE -Run the following command: +On the VM, run the following command: ```bash go run ./cmd/gcp/ --project_id=${GOOGLE_PROJECT} --bucket=${GOOGLE_PROJECT}-${TESSERA_BASE_NAME}-bucket --spanner_db_path=projects/${GOOGLE_PROJECT}/instances/${TESSERA_BASE_NAME}/databases/${TESSERA_BASE_NAME}-db --spanner_db_path=projects/${GOOGLE_PROJECT}/instances/${TESSERA_BASE_NAME}/databases/${TESSERA_BASE_NAME}-dedup-db --private_key=./testdata/ct-http-server.privkey.pem --password=dirk --roots_pem_file=./testdata/fake-ca.cert --origin=${TESSERA_BASE_NAME} ``` From 8ea37907fc313b6f3c48d73aba478454dc820f88 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Tue, 8 Oct 2024 16:20:44 +0000 Subject: [PATCH 3/4] s/go/Go/g --- deployment/live/gcp/test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/live/gcp/test/README.md b/deployment/live/gcp/test/README.md index 5b8e6a0..8acad6c 100644 --- a/deployment/live/gcp/test/README.md +++ b/deployment/live/gcp/test/README.md @@ -2,7 +2,7 @@ ## Prerequisites You'll need to have a VM running in the same GCP project that you can SSH to, -with go installed. +with Go installed. ## Overview From 54412fabc063aa46a4308258b6a6c0ae2eb28741 Mon Sep 17 00:00:00 2001 From: Philippe Boneff Date: Tue, 8 Oct 2024 16:36:16 +0000 Subject: [PATCH 4/4] add todo --- deployment/modules/gcp/storage/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/deployment/modules/gcp/storage/main.tf b/deployment/modules/gcp/storage/main.tf index e3b6bc7..61d2f1c 100644 --- a/deployment/modules/gcp/storage/main.tf +++ b/deployment/modules/gcp/storage/main.tf @@ -9,6 +9,7 @@ terraform { } } +# TODO(phboneff): import tessera terraform directly # Services resource "google_project_service" "serviceusage_googleapis_com" { service = "serviceusage.googleapis.com"