update modis_ndvi test for mindy

ecohealthalliance · Nov 20, 2024 · 3156199 · 3156199
1 parent 70ab928
commit 3156199
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 35 deletions.
diff --git a/R/AWS_get_folder.R b/R/AWS_get_folder.R
@@ -71,6 +71,7 @@ AWS_get_folder <- function(local_folder, ...) {
   downloaded_files <- c()
 
   # Loop through S3 files and download if they don't exist locally
+  # NCL AND THAT THEY CAN BE OPENED!!
   for (file in s3_files) {
 
     # Check if file already exists locally
@@ -187,6 +188,7 @@ AWS_put_files <- function(transformed_file_list,
 
         # Put the file on S3
         s3_upload <- s3$put_object(
+          Body = file.path(local_folder, file),
           Bucket = Sys.getenv("AWS_BUCKET_ID"),
           Key = file.path(local_folder, file))
       }
@@ -195,7 +197,7 @@ AWS_put_files <- function(transformed_file_list,
       # Remove the file from AWS if it's present in the folder and on AWS
       # but not in the list of successfully transformed files. This file is
       # not relevant to the pipeline
-      if(file %in% basename(s3_files)) {
+      if(file.path(local_folder, file) %in% s3_files) {
 
         outcome <- c(outcome, glue::glue("Cleaning up dangling file {file} from AWS"))
 

diff --git a/_targets.R b/_targets.R
@@ -70,7 +70,8 @@ static_targets <- tar_plan(
 
   tar_target(soil_preprocessed_AWS_upload, AWS_put_files(soil_preprocessed, 
                                                          soil_directory),
-             error = "null"), # Continue the pipeline even on error
+             error = "null",
+             cue = tar_cue("always")), # Continue the pipeline even on error
 
   # ASPECT -------------------------------------------------
   tar_target(aspect_urls, c("aspect_zero" = "https://www.fao.org/fileadmin/user_upload/soils/HWSD%20Viewer/GloAspectClN_30as.rar",
@@ -407,7 +408,8 @@ dynamic_targets <- tar_plan(
                                                         modis_ndvi_bundle_request,
                                                         continent_raster_template,
                                                         modis_ndvi_transformed_directory),
-             error = "null"),
+             error = "null",
+             cue = tar_cue("always")),
 
   # Collect branches from modis_ndvi_bundle_request and split into branches 
   # where each branch is a batch of 10 requests
@@ -447,9 +449,10 @@ dynamic_targets <- tar_plan(
              error = "null"), # Repository local means it isn't stored on AWS just yet.
 
   # Put modis_ndvi_transformed files on AWS
-  tar_target(modis_ndvi_transformed_AWS_upload, AWS_put_files(modis_ndvi_transformed,
-                                                              modis_ndvi_transformed_directory),
-             error = "null"),
+  tar_target(modis_ndvi_transformed_AWS_upload, AWS_put_files(transformed_file_list = modis_ndvi_transformed,
+                                                              local_folder = modis_ndvi_transformed_directory),
+             error = "null",
+             cue = tar_cue("always")),
 
   # NASA POWER recorded weather -----------------------------------------------------------
   # RH2M            MERRA-2 Relative Humidity at 2 Meters (%) ;