adding neon download to terrestrial and aquatics

eco4cast · Nov 5, 2023 · f23d003 · f23d003
1 parent 2900527
commit f23d003
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 15 deletions.
diff --git a/targets/aquatics_targets.R b/targets/aquatics_targets.R
@@ -1,18 +1,5 @@
-#renv::restore()
-
 message(paste0("Running Creating Aquatics Targets at ", Sys.time()))
 
-avro_file_directory <- "/home/rstudio/data/aquatic_avro"
-parquet_file_directory <- "/home/rstudio/data/aquatic_parquet"
-EDI_file_directory <- "/home/rstudio/data/aquatic_EDI"
-
-readRenviron("~/.Renviron") # compatible with littler
-Sys.setenv("NEONSTORE_HOME" = "/home/rstudio/data/neonstore")
-
-
-# Sys.setenv("NEONSTORE_HOME" = "/home/rstudio/data/neonstore") #Sys.setenv("NEONSTORE_HOME" = "/efi_neon_challenge/neonstore")
-#Sys.setenv("NEONSTORE_DB" = "home/rstudio/data/neonstore")    #Sys.setenv("NEONSTORE_DB" = "/efi_neon_challenge/neonstore")
-
 Sys.unsetenv("AWS_DEFAULT_REGION")
 Sys.unsetenv("AWS_S3_ENDPOINT")
 Sys.setenv("AWS_EC2_METADATA_DISABLED"="TRUE")
@@ -32,6 +19,35 @@ source('R/data_processing.R')
 
 `%!in%` <- Negate(`%in%`) # not in function
 
+avro_file_directory <- "/home/rstudio/data/aquatic_avro"
+parquet_file_directory <- "/home/rstudio/data/aquatic_parquet"
+EDI_file_directory <- "/home/rstudio/data/aquatic_EDI"
+
+readRenviron("~/.Renviron") # compatible with littler
+Sys.setenv("NEONSTORE_HOME" = "/home/rstudio/data/neonstore")
+Sys.getenv("NEONSTORE_DB")
+
+#temporary aquatic repo during test of new workflow
+site_data <- readr::read_csv("https://raw.githubusercontent.com/eco4cast/neon4cast-targets/main/NEON_Field_Site_Metadata_20220412.csv")
+aq_sites <- site_data |> filter(aquatics == 1) |> pull(field_site_id)
+
+message("aquatics targets")
+neonstore::neon_download("DP1.20288.001", site = aq_sites) # water quality
+neonstore::neon_store(table = "waq_instantaneous", n = 50)
+
+neonstore::neon_download("DP1.20264.001", table ='30', site =  aq_sites)
+neonstore::neon_store(table = "TSD_30_min")
+
+neonstore::neon_download("DP1.20053.001", table ='30', site =  aq_sites)
+neonstore::neon_store(table = "TSW_30min")
+
+neon_disconnect()
+
+# Sys.setenv("NEONSTORE_HOME" = "/home/rstudio/data/neonstore") #Sys.setenv("NEONSTORE_HOME" = "/efi_neon_challenge/neonstore")
+#Sys.setenv("NEONSTORE_DB" = "home/rstudio/data/neonstore")    #Sys.setenv("NEONSTORE_DB" = "/efi_neon_challenge/neonstore")
+
+
+
 message(paste0("Running Creating Aquatics Targets at ", Sys.time()))
 
 sites <- readr::read_csv("NEON_Field_Site_Metadata_20220412.csv") |>
@@ -899,5 +915,4 @@ s3 <- arrow::s3_bucket("bio230014-bucket01/challenges/targets/project_id=neon4ca
 
 arrow::write_csv_arrow(hourly_temp_profile_lakes2, sink = s3$path("aquatics-expanded-observations.csv.gz"))
 
-
 message(paste0("Completed Aquatics Target at ", Sys.time()))
diff --git a/targets/terrestrial_targets.R b/targets/terrestrial_targets.R
@@ -24,6 +24,9 @@ site_names <- sites$field_site_id
 # Terrestrial
 
 #Get the published files on the portal
+neonstore::neon_download(product = "DP4.00200.001", site = site_names, type = "basic")
+neonstore::neon_store(product = "DP4.00200.001")
+#neon_disconnect()
 
 #DP4.00200.001 & DP1.00094.001
 #neon_store(product = "DP4.00200.001")
@@ -335,4 +338,4 @@ s3 <- arrow::s3_bucket("bio230014-bucket01/challenges/targets/project_id=neon4ca
 
 arrow::write_csv_arrow(flux_target_daily2, sink = s3$path("terrestrial_daily-targets.csv.gz"))
 
-
+message(paste0("Completed Terrestrial Target at ", Sys.time()))