change to new gfs nwp-consumer (#732)

* change to new gfs nwp-consumer * default = "1.0.10"
openclimatefix · Jan 10, 2025 · a16bf13 · a16bf13
1 parent 07293d3
commit a16bf13
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 14 deletions.
diff --git a/terraform/india/development/main.tf b/terraform/india/development/main.tf
@@ -127,8 +127,8 @@ module "nwp_consumer_ecmwf_live_ecs_task" {
   ecs-task_type               = "consumer"
   ecs-task_execution_role_arn = module.ecs-cluster.ecs_task_execution_role_arn
   ecs-task_size = {
-      cpu    = 2048
-      memory = 10240
+      cpu    = 512
+      memory = 1024
   }
 
   aws-region                    = var.region
@@ -176,8 +176,8 @@ module "nwp_consumer_gfs_live_ecs_task" {
   ecs-task_name               = "nwp-consumer-gfs-india"
   ecs-task_type               = "consumer"
   ecs-task_size = {
-    cpu    = 1024
-    memory = 5120
+    cpu    = 512
+    memory = 1024
   }
   ecs-task_execution_role_arn = module.ecs-cluster.ecs_task_execution_role_arn
 
@@ -192,23 +192,21 @@ module "nwp_consumer_gfs_live_ecs_task" {
   ]
 
   container-env_vars = [
+    { "name" : "MODEL_REPOSITORY", "value" : "gfs" },
     { "name" : "AWS_REGION", "value" : var.region },
-    { "name" : "AWS_S3_BUCKET", "value" : module.s3-nwp-bucket.bucket_id },
+    { "name" : "ZARRDIR", "value" : "s3://${module.s3-nwp-bucket.bucket_id}/gfs/data" },
     { "name" : "LOGLEVEL", "value" : "DEBUG" },
     { "name" : "SENTRY_DSN", "value" : var.sentry_dsn },
+    { "name" : "CONCURRENCY", "value" : "false" },
+    # legacy
+    { "name" : "AWS_S3_BUCKET", "value" : module.s3-nwp-bucket.bucket_id },
     { "name" : "ENVIRONMENT", "value" : local.environment },
   ]
   container-secret_vars = []
-  container-tag         = var.version-nwp
+  container-tag         = var.version-nwp-gfs
   container-name        = "openclimatefix/nwp-consumer"
   container-command     = [
-    "download",
-    "--source=gfs",
-    "--sink=s3",
-    "--rdir=gfs/raw",
-    "--zdir=gfs/data",
-    "--create-latest",
-    "--no-rename-vars"
+    "consume"
   ]
 }
 

diff --git a/terraform/india/development/variables.tf b/terraform/india/development/variables.tf
@@ -16,6 +16,12 @@ variable version-nwp-ecmwf {
   description = "Container image tag of the NWP consumer to use for ECMWF: openclimatefix/nwp-consumer"
 }
 
+variable version-nwp-gfs {
+  type = string
+  default = "1.0.10"
+  description = "Container image tag of the NWP consumer to use for GFS: openclimatefix/nwp-consumer"
+}
+
 variable version-india_api {
   type = string
   default = "0.1.0"

diff --git a/terraform/modules/services/airflow/dags/india/nwp-dag.py b/terraform/modules/services/airflow/dags/india/nwp-dag.py
@@ -91,7 +91,11 @@
         task_id="determine_latest_zarr_ecmwf",
     )(bucket=f'india-nwp-{env}', prefix='ecmwf/data')
 
+    rename_zarr_gfs = determine_latest_zarr.override(
+        task_id="determine_latest_zarr_gfs",
+    )(bucket=f'india-nwp-{env}', prefix='gfs/data')
+
     latest_only >> nwp_consumer_ecmwf >> rename_zarr_ecmwf
-    latest_only >> nwp_consumer_gfs
+    latest_only >> nwp_consumer_gfs >> rename_zarr_gfs
     latest_only >> nwp_consumer_metoffice >> rename_zarr_metoffice