diff --git a/Dockerfile b/Dockerfile index 4cde058..ad6994b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,4 +6,4 @@ ENV FLEX_TEMPLATE_JAVA_CLASSPATH=/template/* ENV FLEX_TEMPLATE_JAVA_MAIN_CLASS=io.blockchainetl.ethereum.EthereumPubSubToBigQueryPipeline COPY target/blockchain-etl-dataflow-bundled-0.1.jar /template/ -COPY chain-config/blockchain_zkevm_imtbl_testnet_${env}.json /template/ \ No newline at end of file +COPY chain-config/blockchain_*_${env}.json /template/ \ No newline at end of file diff --git a/chain-config/blockchain_zkevm_imtbl_mainnet_dev.json b/chain-config/blockchain_zkevm_imtbl_mainnet_dev.json new file mode 100644 index 0000000..b79f1db --- /dev/null +++ b/chain-config/blockchain_zkevm_imtbl_mainnet_dev.json @@ -0,0 +1,8 @@ +[ + { + "transformNamePrefix": "blockchain_zkevm_imtbl_mainnet_", + "pubSubSubscriptionPrefix": "projects/dev-im-data/subscriptions/blockchain-zkevm-imtbl-mainnet-dataflow", + "bigQueryDataset": "raw_blockchain_zkevm_imtbl_mainnet", + "startTimestamp": "2019-03-02T00:00:00Z" + } +] diff --git a/chain-config/blockchain_zkevm_imtbl_mainnet_prod.json b/chain-config/blockchain_zkevm_imtbl_mainnet_prod.json new file mode 100644 index 0000000..e5f2e16 --- /dev/null +++ b/chain-config/blockchain_zkevm_imtbl_mainnet_prod.json @@ -0,0 +1,8 @@ +[ + { + "transformNamePrefix": "blockchain_zkevm_imtbl_mainnet_", + "pubSubSubscriptionPrefix": "projects/prod-im-data/subscriptions/blockchain-zkevm-imtbl-mainnet-dataflow", + "bigQueryDataset": "raw_blockchain_zkevm_imtbl_mainnet", + "startTimestamp": "2019-03-02T00:00:00Z" + } +] diff --git a/terraform/main.tf b/terraform/main.tf index 9a474ec..74886df 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -97,4 +97,31 @@ resource "google_dataflow_flex_template_job" "flex_template_job" { region=local.region zone=local.zone } +} + + +resource "google_dataflow_flex_template_job" "zkevm_imtbl_mainnet_job" { + provider = google-beta + project = "${terraform.workspace}-im-data" + name = "zkevm-imtbl-mainnet-etl-dataflow-${formatdate("YYYYMMDD-hhmmss", timestamp())}" + region = local.region + skip_wait_on_job_termination = true + container_spec_gcs_path = "gs://${google_storage_bucket_object.blockchain_etl_dataflow.bucket}/${google_storage_bucket_object.blockchain_etl_dataflow.name}" + on_delete = "drain" + + + parameters = { + chainConfigFile = "/template/blockchain_zkevm_imtbl_mainnet_${terraform.workspace}.json" + allowedTimestampSkewSeconds = "5184000" + gcpTempLocation = "gs://${terraform.workspace}-im-data-imx-resource/ethereum-etl/zkevm-imtbl-mainnet-streaming/temp" + tempLocation = "gs://${terraform.workspace}-im-data-imx-resource/ethereum-etl/zkevm-imtbl-mainnet-streaming/temp" + project = "${terraform.workspace}-im-data" + runner = "DataflowRunner" + workerMachineType = "n1-standard-1" + maxNumWorkers = 1 + diskSizeGb=30 + enableStreamingEngine=true + region=local.region + zone=local.zone + } } \ No newline at end of file diff --git a/terraform/version.tf b/terraform/version.tf index 2040adb..132c9d2 100644 --- a/terraform/version.tf +++ b/terraform/version.tf @@ -9,7 +9,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 4.0.0" + version = "~> 4.82.0, < 5.0.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 4.82.0, < 5.0.0" } } provider_meta "google" {