Merge pull request #5 from immutable/d1155

DATA-1181:Add config for imtbl zkevm mainnet

Dockerfile

@@ -6,4 +6,4 @@ ENV FLEX_TEMPLATE_JAVA_CLASSPATH=/template/*
 ENV FLEX_TEMPLATE_JAVA_MAIN_CLASS=io.blockchainetl.ethereum.EthereumPubSubToBigQueryPipeline
 
 COPY target/blockchain-etl-dataflow-bundled-0.1.jar /template/
-COPY chain-config/blockchain_zkevm_imtbl_testnet_${env}.json /template/
+COPY chain-config/blockchain_*_${env}.json /template/

chain-config/blockchain_zkevm_imtbl_mainnet_dev.json

@@ -0,0 +1,8 @@
+[
+    {
+        "transformNamePrefix": "blockchain_zkevm_imtbl_mainnet_",
+        "pubSubSubscriptionPrefix": "projects/dev-im-data/subscriptions/blockchain-zkevm-imtbl-mainnet-dataflow",
+        "bigQueryDataset": "raw_blockchain_zkevm_imtbl_mainnet",
+        "startTimestamp": "2019-03-02T00:00:00Z"
+    }
+]

chain-config/blockchain_zkevm_imtbl_mainnet_prod.json

@@ -0,0 +1,8 @@
+[
+    {
+        "transformNamePrefix": "blockchain_zkevm_imtbl_mainnet_",
+        "pubSubSubscriptionPrefix": "projects/prod-im-data/subscriptions/blockchain-zkevm-imtbl-mainnet-dataflow",
+        "bigQueryDataset": "raw_blockchain_zkevm_imtbl_mainnet",
+        "startTimestamp": "2019-03-02T00:00:00Z"
+    }
+]

terraform/main.tf

@@ -97,4 +97,31 @@ resource "google_dataflow_flex_template_job" "flex_template_job" {
     region=local.region
     zone=local.zone
   }
+}
+
+
+resource "google_dataflow_flex_template_job" "zkevm_imtbl_mainnet_job" {
+  provider                     = google-beta
+  project                      = "${terraform.workspace}-im-data"
+  name                         = "zkevm-imtbl-mainnet-etl-dataflow-${formatdate("YYYYMMDD-hhmmss", timestamp())}"
+  region                       = local.region
+  skip_wait_on_job_termination = true
+  container_spec_gcs_path      = "gs://${google_storage_bucket_object.blockchain_etl_dataflow.bucket}/${google_storage_bucket_object.blockchain_etl_dataflow.name}"
+  on_delete                    = "drain"
+
+
+  parameters = {
+    chainConfigFile = "/template/blockchain_zkevm_imtbl_mainnet_${terraform.workspace}.json"
+    allowedTimestampSkewSeconds = "5184000"
+    gcpTempLocation = "gs://${terraform.workspace}-im-data-imx-resource/ethereum-etl/zkevm-imtbl-mainnet-streaming/temp"
+    tempLocation = "gs://${terraform.workspace}-im-data-imx-resource/ethereum-etl/zkevm-imtbl-mainnet-streaming/temp"
+    project = "${terraform.workspace}-im-data"
+    runner = "DataflowRunner"
+    workerMachineType = "n1-standard-1"
+    maxNumWorkers = 1
+    diskSizeGb=30
+    enableStreamingEngine=true
+    region=local.region
+    zone=local.zone
+  }
 }

terraform/version.tf

@@ -9,7 +9,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 4.0.0"
+      version = "~> 4.82.0, < 5.0.0"
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = ">= 4.82.0, < 5.0.0"
     }
   }
   provider_meta "google" {