Merge pull request #693 from ldecicco-USGS/main

Messaging and WQP order

.Rbuildignore

@@ -1,4 +1,5 @@
 ^Meta$
+^Temp$
 ^doc$
 .Rhistory
 vignettes/figures
@@ -88,5 +89,6 @@ vignettes/github_code.png
 vignettes/save_pat.png
 vignettes/more_tests.png
 vignettes/stylesSlides.css
+vignettes/Status.Rmd
 ^ci$
 ^public$

.gitignore

@@ -11,3 +11,4 @@ docs
 .DS_Store
 /doc/
 /Meta/
+/Temp/

NEWS

@@ -1,6 +1,8 @@
 dataRetrieval 2.7.15
 ===================
 * Update warning about upcoming changes to USGS water quality data.
+* Added date sorting to all WQP functions.
+
 
 dataRetrieval 2.7.14
 ===================

R/AAA.R

@@ -30,3 +30,12 @@ If you have additional questions about these changes,
 email [email protected].")
 }
 
+nwis_message <- function(){
+  return("WARNING: Beginning in late February 2024 the NWIS qw data endpoint will not deliver new data or updates to existing data. 
+Eventually the endpoint will be retired. For updated information visit:
+https://waterdata.usgs.gov/nwis/qwdata
+For additional details, see vignettes:
+https://doi-usgs.github.io/dataRetrieval/articles/Status.html
+https://doi-usgs.github.io/dataRetrieval/articles/qwdata_changes.html
+If you have additional questions about the qw data service, email [email protected].")
+}

R/citations.R

@@ -28,7 +28,7 @@ create_NWIS_bib <- function(x){
                         format(attr(x, "queryTime"), "%b %d, %Y"),
                         ", at ",
                         attr(x, "url"),
-                        ", http://dx.doi.org/10.5066/F7P55KJN")
+                        ", https://dx.doi.org/10.5066/F7P55KJN")
 
   ref <- utils::bibentry(
     bibtype = "Manual",

R/dataRetrievals-package.R

@@ -21,7 +21,7 @@ Extended Documentation: https://doi-usgs.github.io/dataRetrieval")
 #'
 #' Retrieval functions for USGS and EPA hydrologic and water quality data.
 #'
-#' Please see \url{https://pubs.usgs.gov/publication/tm4A10} for more information.
+#' Please see \url{https://doi-usgs.github.io/dataRetrieval/} for more information.
 #'
 #' @name dataRetrieval
 #' @aliases dataRetrieval-package

R/importWQP.R

@@ -269,6 +269,8 @@ importWQP <- function(obs_url, zip = TRUE, tz = "UTC",
         retval$ActivityStartDateTime <- lubridate::fast_strptime(retval$ActivityStartDateTime, "%Y-%m-%d %H:%M:%S") +
           60 * 60 * retval$timeZoneStart
         attr(retval$ActivityStartDateTime, "tzone") <- tz
+        # if we're going to sort, here's where we'd do it:
+        retval <- retval[order(retval$ActivityStartDateTime),]
       }
 
       if (all(c("ActivityEndDate", "ActivityEndTime/Time") %in% names(retval))) {

R/readNWISqw.R

@@ -106,13 +106,7 @@ readNWISqw <- function(siteNumbers,
                        tz = "UTC") {
   .Deprecated(
     new = "readWQPqw", package = "dataRetrieval",
-    msg = "WARNING: Beginning in late February 2024 the NWIS qw data endpoint will not deliver new data or updates to existing data. 
-Eventually the endpoint will be retired. For updated information visit:
-https://waterdata.usgs.gov/nwis/qwdata
-For additional details, see vignettes:
-https://doi-usgs.github.io/dataRetrieval/articles/Status.html
-https://doi-usgs.github.io/dataRetrieval/articles/qwdata_changes.html
-If you have additional questions about the qw data service, email [email protected]."
+    msg = nwis_message()
   )
 
   pgrp <- c(

R/readNWISunit.R

@@ -642,7 +642,7 @@ readNWISstat <- function(siteNumbers, parameterCd, startDate = "", endDate = "",
 #' @param stateCd could be character (full name, abbreviation, id), or numeric (id).
 #' Only one is accepted per query.
 #' @param countyCd could be character (name, with or without "County", or "ALL"),
-#' numeric (id), or code{NULL}, which will
+#' numeric (id), or \code{NULL}, which will
 #' return state or national data depending on the stateCd argument.  "ALL" may
 #' also be supplied, which will return data
 #' for every county in a state. Can be a vector of counties in the same state.

R/whatNWISdata.R

@@ -86,16 +86,17 @@
 whatNWISdata <- function(..., convertType = TRUE) {
   matchReturn <- convertLists(...)
 
+  prewarned <- FALSE
   if ("service" %in% names(matchReturn)) {
     service <- matchReturn$service
 
     if (any(service %in% c("qw", "qwdata"))) {
       .Deprecated(
         old = "whatNWISdata", package = "dataRetrieval",
         new = "whatWQPdata",
-        msg = "NWIS qw web services are being retired. Please see the vignette
-'Changes to NWIS QW services' for more information."
+        msg = nwis_message()
       )
+      prewarned <- TRUE
     } 
   } else {
     service <- "all"
@@ -155,5 +156,10 @@ whatNWISdata <- function(..., convertType = TRUE) {
     SiteFile$end_date <- as.Date(suppressWarnings(lubridate::parse_date_time(SiteFile$end_date, c("Ymd", "mdY", "Y!"))))
   }
 
+  if(any(SiteFile$data_type_cd == "qw")){
+    if(!prewarned){
+      message(nwis_message())
+    }
+  }
   return(SiteFile)
 }

vignettes/Status.Rmd

@@ -37,34 +37,85 @@ These profiles are not yet available to test. When they are, we will update this
 
 
 
-# Current Status: dataRetrieval specific
+# What to expect: dataRetrieval specific
 
-### readNWISqw
+## readNWISqw
+
+### Current Status:
 
 Functioning normally, includes new data. This function will be going away soon, so consider switching to `readWQPqw` or `readWQPdata`. When the NWIS qw service is eventually shut down - the `readNWISqw` will be removed from dataRetrieval. 
 
-* What to expect:
+### Late February 2024
+
+When the NWIS services are "frozen": the returned data will not include any new data. 
+
+### Fall 2024
 
-See `vignette("qwdata_changes", package = "dataRetrieval")` for information on how to convert your workflows from `readNWISqw` to `readWQPqw`. This function will be removed from the dataRetrieval package when the NWIS services are decommissioned . 
+When the NWIS services are decommissioned (possible September 2024): the `readNWISqw` function will be removed from `dataRetrieval`.
 
+See `vignette("qwdata_changes", package = "dataRetrieval")` for information on how to convert your workflows from `readNWISqw` to `readWQPqw`. 
 
-### whatNWISdata
+## whatNWISdata
+
+### Current Status:
 
 Functioning normally, includes information on all available data. 
 
-*  What to expect: 
+### Late February 2024
+
+When the NWIS services are "frozen" (late February): the returned data availability will also be frozen ONLY for "qw" data_type_cd results. All other data types should not be affected. 
+
+### Fall 2024
+
+When the NWIS services are decommissioned (possible September 2024): there will no longer be any "qw" information provided in the output of `whatNWISdata`. Discrete water-quality availability will be available via WQP services. More information will be provided as we learn more.
+
+Here's an example of what will change:
+
+```{r eval=FALSE}
+what_NWIS <- whatNWISdata(siteNumber = "05114000")
+nrow(what_NWIS)
+[1] 407
+nrow(what_NWIS[what_NWIS$data_type_cd == "qw",])
+[1] 381
+```
 
-When the USGS data is "frozen" (late Feburary), the returned data availablity will also be frozen ONLY for "qw" data_type_cd results. All other data types should not be affected. 
+So for site "05114000", there are 381 NWIS qw parameters that have been measured. Starting late February 2024, the data availability for those 381 parameters will be frozen...even if new data is collected. Eventually those 381 rows of data will not be returned, only 26 rows of data will be returned (407-381). 
 
-When the NWIS services are decommissioned, there will no longer be any "qw" information provided in the output of `whatNWISdata`. Discrete water-quality availability will be available via WQP services. More information will be provided as we learn more.
+Once the full transition is completed, we will update with additional guidance. Water Quality Portal data availability can be obtained:
 
-### readWQP...
+```{r eval=FALSE}
+what_WQP_full <- readWQPsummary(siteid = "USGS-05114000")
+
+library(dplyr)
+what_WQP <- what_WQP_full |> 
+  group_by(siteid = MonitoringLocationIdentifier, CharacteristicName) |> 
+  summarise(count = sum(ResultCount, na.rm = TRUE),
+            start_year = min(YearSummarized, na.rm = TRUE),
+            end_year = max(YearSummarized, na.rm = TRUE))
+
+nrow(what_WQP)
+[1] 203
+```
+
+You'll notice that there are fewer rows in the `what_WQP` data frame (203) than "qw" data in the `what_NWIS` (381). This is because there are sometimes more than one parameter code that are associated with a "CharacteristicName". 
+
+## readWQPqw and readWQPdata
+
+### Current Status:
 
 Functioning normally, includes all available USGS data. The Water Quality Portal does not yet include new "WQX 3.0" profiles. 
 
-*  What to expect: 
+### Late February 2024
+
+When the NWIS services are "frozen" (late February): no new USGS will be available via any Water Quality Portal query. 
+
+### Fall 2024
+
+When the NWIS services are decommissioned (possible September 2024): Should not affect WQP functions.
+
+### March/April 2024
 
-USGS data will be "frozen" around late February, and no new USGS will be available. Late March/early April there will be new data profiles that will include ALL USGS data (legacy and new). Shortly after we plan to update dataRetrieval so that these new profiles are the default. 
+When WQP gets new "data profiles" (late March/April): New data profiles will include ALL USGS data (legacy and new). Shortly after these profiles are released, we plan to update dataRetrieval so that these new profiles are the default. 
 
 # Known Dependencies
 

vignettes/dataRetrieval.Rmd

@@ -722,33 +722,128 @@ citation(package = "dataRetrieval")
 
 ## Citing NWIS data
 
-U.S. Geological Survey, 2023, National Water Information System data available on the World Wide Web (USGS Water Data for the Nation), accessed [April 26, 2023], at http://waterdata.usgs.gov/nwis/. http://dx.doi.org/10.5066/F7P55KJN
+The requested format is: 
 
-This can be done using the `create_NWIS_bib` function:
+U.S. Geological Survey, 2023, National Water Information System data available on the World Wide Web (USGS Water Data for the Nation), accessed [April 26, 2023], at https://waterdata.usgs.gov/nwis/. https://dx.doi.org/10.5066/F7P55KJN
+
+This can be created from the attributes of the data using the `create_NWIS_bib` function:
 
 
 ```{r nwisCite, eval=FALSE}
 
 dv <- readNWISdv("09010500", "00060")
 
 NWIScitation <- create_NWIS_bib(dv)
+NWIScitation
+```
+
+```
+U.S. Geological Survey (2024). _National Water Information
+System data available on the World Wide Web (USGS Water Data
+for the Nation)_. doi:10.5066/F7P55KJN
+<https://doi.org/10.5066/F7P55KJN>, Accessed Feb 09, 2024,
+<https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml,1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01>.
+```
+Alternative output is available:
+
+```{r show1, eval=FALSE}
+print(NWIScitation, style = "Bibtex")
+```
+
+```
+@Manual{,
+  title = {National Water Information System data available on the World Wide Web (USGS Water Data for the Nation)},
+  author = {{U.S. Geological Survey}},
+  doi = {10.5066/F7P55KJN},
+  note = {Accessed Feb 09, 2024},
+  year = {2024},
+  url = {https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml,1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01},
+}
+```
+
+```{r show2, eval=FALSE}
+print(NWIScitation, style = "citation")
+```
 
 ```
+U.S. Geological Survey, 2024, National Water Information System
+data available on the World Wide Web (USGS Water Data for the
+Nation), accessed Feb 09, 2024, at
+https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml,1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01,
+https://dx.doi.org/10.5066/F7P55KJN
+
+A BibTeX entry for LaTeX users is
+
+  @Manual{,
+    title = {National Water Information System data available on the World Wide Web (USGS Water Data for the Nation)},
+    author = {{U.S. Geological Survey}},
+    doi = {10.5066/F7P55KJN},
+    note = {Accessed Feb 09, 2024},
+    year = {2024},
+    url = {https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml,1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01},
+  }
+```
+
+
 
 ## Citing WQP data
 
 Citations for specific datasets should use this format:
 
 National Water Quality Monitoring Council, YYYY, Water Quality Portal, accessed mm, dd, yyyy, hyperlink_for_query, https://doi.org/10.5066/P9QRKUVJ.
 
-This can be done using the `create_WQP_bib` function:
+This can be obtained using the `create_WQP_bib` function:
 
 ```{r WQPcite, eval=FALSE}
 SC <- readWQPqw(siteNumbers = "USGS-05288705",
                 parameterCd = "00300")
 
 WQPcitation <- create_WQP_bib(SC)
+WQPcitation
+```
+
+```
+National Water Quality Monitoring Council (2024). _ Water
+Quality Portal_. doi:10.5066/P9QRKUVJ
+<https://doi.org/10.5066/P9QRKUVJ>, Accessed Feb 09, 2024,
+<https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&pCode=00300&mimeType=tsv&zip=yes>.
+```
+
+```{r show3, eval=FALSE}
+print(WQPcitation, style = "Bibtex")
+```
+
+```
+@Manual{,
+  title = { Water Quality Portal},
+  author = {{National Water Quality Monitoring Council}},
+  doi = {10.5066/P9QRKUVJ},
+  note = {Accessed Feb 09, 2024},
+  year = {2024},
+  url = {https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&pCode=00300&mimeType=tsv&zip=yes},
+}
+```
+
+```{r show4, eval=FALSE}
+print(WQPcitation, style = "citation")
+```
+
+```
+National Water Quality Monitoring Council, 2024, Water Quality
+Portal, accessed 02, 09, 2024,
+https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&pCode=00300&mimeType=tsv&zip=yes,
+https://doi.org/10.5066/P9QRKUVJ.
+
+A BibTeX entry for LaTeX users is
 
+  @Manual{,
+    title = { Water Quality Portal},
+    author = {{National Water Quality Monitoring Council}},
+    doi = {10.5066/P9QRKUVJ},
+    note = {Accessed Feb 09, 2024},
+    year = {2024},
+    url = {https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&pCode=00300&mimeType=tsv&zip=yes},
+  }
 ```
 
 ## Citing Water Quality Portal itself