From 33cbcb4166d1cb6e70dc52d1a5f39040297c97a9 Mon Sep 17 00:00:00 2001 From: Laura A DeCicco Date: Fri, 9 Feb 2024 13:13:10 -0600 Subject: [PATCH 1/7] Adding more citation info --- vignettes/dataRetrieval.Rmd | 99 ++++++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 2 deletions(-) diff --git a/vignettes/dataRetrieval.Rmd b/vignettes/dataRetrieval.Rmd index 34b6ad3b..f44943a9 100644 --- a/vignettes/dataRetrieval.Rmd +++ b/vignettes/dataRetrieval.Rmd @@ -722,9 +722,11 @@ citation(package = "dataRetrieval") ## Citing NWIS data +The requested format is: + U.S. Geological Survey, 2023, National Water Information System data available on the World Wide Web (USGS Water Data for the Nation), accessed [April 26, 2023], at http://waterdata.usgs.gov/nwis/. http://dx.doi.org/10.5066/F7P55KJN -This can be done using the `create_NWIS_bib` function: +This can be created from the attributes of the data using the `create_NWIS_bib` function: ```{r nwisCite, eval=FALSE} @@ -732,8 +734,57 @@ This can be done using the `create_NWIS_bib` function: dv <- readNWISdv("09010500", "00060") NWIScitation <- create_NWIS_bib(dv) +NWIScitation +``` + +``` +U.S. Geological Survey (2024). _National Water Information +System data available on the World Wide Web (USGS Water Data +for the Nation)_. doi:10.5066/F7P55KJN +, Accessed Feb 09, 2024, +. +``` +Alternative output is available: + +```{r show1, eval=FALSE} +print(NWIScitation, style = "Bibtex") +``` + +``` +@Manual{, + title = {National Water Information System data available on the World Wide Web (USGS Water Data for the Nation)}, + author = {{U.S. Geological Survey}}, + doi = {10.5066/F7P55KJN}, + note = {Accessed Feb 09, 2024}, + year = {2024}, + url = {https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml,1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01}, +} +``` + +```{r show2, eval=FALSE} +print(NWIScitation, style = "citation") +``` ``` +U.S. Geological Survey, 2024, National Water Information System +data available on the World Wide Web (USGS Water Data for the +Nation), accessed Feb 09, 2024, at +https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml,1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01, +http://dx.doi.org/10.5066/F7P55KJN + +A BibTeX entry for LaTeX users is + + @Manual{, + title = {National Water Information System data available on the World Wide Web (USGS Water Data for the Nation)}, + author = {{U.S. Geological Survey}}, + doi = {10.5066/F7P55KJN}, + note = {Accessed Feb 09, 2024}, + year = {2024}, + url = {https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml,1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01}, + } +``` + + ## Citing WQP data @@ -741,14 +792,58 @@ Citations for specific datasets should use this format: National Water Quality Monitoring Council, YYYY, Water Quality Portal, accessed mm, dd, yyyy, hyperlink_for_query, https://doi.org/10.5066/P9QRKUVJ. -This can be done using the `create_WQP_bib` function: +This can be obtained using the `create_WQP_bib` function: ```{r WQPcite, eval=FALSE} SC <- readWQPqw(siteNumbers = "USGS-05288705", parameterCd = "00300") WQPcitation <- create_WQP_bib(SC) +WQPcitation +``` + +``` +National Water Quality Monitoring Council (2024). _ Water +Quality Portal_. doi:10.5066/P9QRKUVJ +, Accessed Feb 09, 2024, +. +``` + +```{r show3, eval=FALSE} +print(WQPcitation, style = "Bibtex") +``` + +``` +@Manual{, + title = { Water Quality Portal}, + author = {{National Water Quality Monitoring Council}}, + doi = {10.5066/P9QRKUVJ}, + note = {Accessed Feb 09, 2024}, + year = {2024}, + url = {https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&pCode=00300&mimeType=tsv&zip=yes}, +} +``` + +```{r show4, eval=FALSE} +print(WQPcitation, style = "citation") +``` + +``` +National Water Quality Monitoring Council, 2024, Water Quality +Portal, accessed 02, 09, 2024, +https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&pCode=00300&mimeType=tsv&zip=yes, +https://doi.org/10.5066/P9QRKUVJ. + +A BibTeX entry for LaTeX users is + @Manual{, + title = { Water Quality Portal}, + author = {{National Water Quality Monitoring Council}}, + doi = {10.5066/P9QRKUVJ}, + note = {Accessed Feb 09, 2024}, + year = {2024}, + url = {https://www.waterqualitydata.us/data/Result/search?siteid=USGS-05288705&pCode=00300&mimeType=tsv&zip=yes}, + } ``` ## Citing Water Quality Portal itself From 8aa8a1cb169d9a90793f0f0fbb2011113692cf0a Mon Sep 17 00:00:00 2001 From: Laura A DeCicco Date: Fri, 9 Feb 2024 14:43:25 -0600 Subject: [PATCH 2/7] More cleanup on messaging --- R/AAA.R | 9 ++++++ R/readNWISqw.R | 8 +---- R/whatNWISdata.R | 10 ++++-- vignettes/Status.Rmd | 73 +++++++++++++++++++++++++++++++++++++------- 4 files changed, 80 insertions(+), 20 deletions(-) diff --git a/R/AAA.R b/R/AAA.R index c73d5544..9925567e 100644 --- a/R/AAA.R +++ b/R/AAA.R @@ -30,3 +30,12 @@ If you have additional questions about these changes, email CompTools@usgs.gov.") } +nwis_message <- function(){ + return("WARNING: Beginning in late February 2024 the NWIS qw data endpoint will not deliver new data or updates to existing data. +Eventually the endpoint will be retired. For updated information visit: +https://waterdata.usgs.gov/nwis/qwdata +For additional details, see vignettes: +https://doi-usgs.github.io/dataRetrieval/articles/Status.html +https://doi-usgs.github.io/dataRetrieval/articles/qwdata_changes.html +If you have additional questions about the qw data service, email CompTools@usgs.gov.") +} diff --git a/R/readNWISqw.R b/R/readNWISqw.R index d030e154..ad7ae36f 100644 --- a/R/readNWISqw.R +++ b/R/readNWISqw.R @@ -106,13 +106,7 @@ readNWISqw <- function(siteNumbers, tz = "UTC") { .Deprecated( new = "readWQPqw", package = "dataRetrieval", - msg = "WARNING: Beginning in late February 2024 the NWIS qw data endpoint will not deliver new data or updates to existing data. -Eventually the endpoint will be retired. For updated information visit: -https://waterdata.usgs.gov/nwis/qwdata -For additional details, see vignettes: -https://doi-usgs.github.io/dataRetrieval/articles/Status.html -https://doi-usgs.github.io/dataRetrieval/articles/qwdata_changes.html -If you have additional questions about the qw data service, email CompTools@usgs.gov." + msg = nwis_message() ) pgrp <- c( diff --git a/R/whatNWISdata.R b/R/whatNWISdata.R index 6d27f15e..324330df 100644 --- a/R/whatNWISdata.R +++ b/R/whatNWISdata.R @@ -86,6 +86,7 @@ whatNWISdata <- function(..., convertType = TRUE) { matchReturn <- convertLists(...) + prewarned <- FALSE if ("service" %in% names(matchReturn)) { service <- matchReturn$service @@ -93,9 +94,9 @@ whatNWISdata <- function(..., convertType = TRUE) { .Deprecated( old = "whatNWISdata", package = "dataRetrieval", new = "whatWQPdata", - msg = "NWIS qw web services are being retired. Please see the vignette -'Changes to NWIS QW services' for more information." + msg = nwis_message() ) + prewarned <- TRUE } } else { service <- "all" @@ -155,5 +156,10 @@ whatNWISdata <- function(..., convertType = TRUE) { SiteFile$end_date <- as.Date(suppressWarnings(lubridate::parse_date_time(SiteFile$end_date, c("Ymd", "mdY", "Y!")))) } + if(any(SiteFile$data_type_cd == "qw")){ + if(!prewarned){ + message(nwis_message()) + } + } return(SiteFile) } diff --git a/vignettes/Status.Rmd b/vignettes/Status.Rmd index 3810192c..8a319d03 100644 --- a/vignettes/Status.Rmd +++ b/vignettes/Status.Rmd @@ -37,34 +37,85 @@ These profiles are not yet available to test. When they are, we will update this -# Current Status: dataRetrieval specific +# What to expect: dataRetrieval specific -### readNWISqw +## readNWISqw + +### Current Status: Functioning normally, includes new data. This function will be going away soon, so consider switching to `readWQPqw` or `readWQPdata`. When the NWIS qw service is eventually shut down - the `readNWISqw` will be removed from dataRetrieval. -* What to expect: +### Late February 2024 + +When the NWIS services are "frozen": the returned data will not include any new data. + +### Fall 2024 -See `vignette("qwdata_changes", package = "dataRetrieval")` for information on how to convert your workflows from `readNWISqw` to `readWQPqw`. This function will be removed from the dataRetrieval package when the NWIS services are decommissioned . +When the NWIS services are decommissioned (possible September 2024): the `readNWISqw` function will be removed from `dataRetrieval`. +See `vignette("qwdata_changes", package = "dataRetrieval")` for information on how to convert your workflows from `readNWISqw` to `readWQPqw`. -### whatNWISdata +## whatNWISdata + +### Current Status: Functioning normally, includes information on all available data. -* What to expect: +### Late February 2024 + +When the NWIS services are "frozen" (late February): the returned data availability will also be frozen ONLY for "qw" data_type_cd results. All other data types should not be affected. + +### Fall 2024 + +When the NWIS services are decommissioned (possible September 2024): there will no longer be any "qw" information provided in the output of `whatNWISdata`. Discrete water-quality availability will be available via WQP services. More information will be provided as we learn more. + +Here's an example of what will change: + +```{r eval=FALSE} +what_NWIS <- whatNWISdata(siteNumber = "05114000") +nrow(what_NWIS) +[1] 407 +nrow(what_NWIS[what_NWIS$data_type_cd == "qw",]) +[1] 381 +``` -When the USGS data is "frozen" (late Feburary), the returned data availablity will also be frozen ONLY for "qw" data_type_cd results. All other data types should not be affected. +So for site "05114000", there are 381 NWIS qw parameters that have been measured. Starting late February 2024, the data availability for those 381 parameters will be frozen...even if new data is collected. Eventually those 381 rows of data will not be returned, only 26 rows of data will be returned (407-381). -When the NWIS services are decommissioned, there will no longer be any "qw" information provided in the output of `whatNWISdata`. Discrete water-quality availability will be available via WQP services. More information will be provided as we learn more. +Once the full transition is completed, we will update with additional guidance. Water Quality Portal data availability can be obtained: -### readWQP... +```{r eval=FALSE} +what_WQP_full <- readWQPsummary(siteid = "USGS-05114000") + +library(dplyr) +what_WQP <- what_WQP_full |> + group_by(siteid = MonitoringLocationIdentifier, CharacteristicName) |> + summarise(count = sum(ResultCount, na.rm = TRUE), + start_year = min(YearSummarized, na.rm = TRUE), + end_year = max(YearSummarized, na.rm = TRUE)) + +nrow(what_WQP) +[1] 203 +``` + +You'll notice that there are fewer rows in the `what_WQP` data frame (203) than "qw" data in the `what_NWIS` (381). This is because there are sometimes more than one parameter code that are associated with a "CharacteristicName". + +## readWQPqw and readWQPdata + +### Current Status: Functioning normally, includes all available USGS data. The Water Quality Portal does not yet include new "WQX 3.0" profiles. -* What to expect: +### Late February 2024 + +When the NWIS services are "frozen" (late February): no new USGS will be available via any Water Quality Portal query. + +### Fall 2024 + +When the NWIS services are decommissioned (possible September 2024): Should not affect WQP functions. + +### March/April 2024 -USGS data will be "frozen" around late February, and no new USGS will be available. Late March/early April there will be new data profiles that will include ALL USGS data (legacy and new). Shortly after we plan to update dataRetrieval so that these new profiles are the default. +When WQP gets new "data profiles" (late March/April): New data profiles will include ALL USGS data (legacy and new). Shortly after these profiles are released, we plan to update dataRetrieval so that these new profiles are the default. # Known Dependencies From 9f14e1d0b3056fa9e1142f5181c439f82d7c4891 Mon Sep 17 00:00:00 2001 From: Laura A DeCicco Date: Fri, 9 Feb 2024 14:59:05 -0600 Subject: [PATCH 3/7] Order WQP data if it has a ActivityStartDateTime --- NEWS | 2 ++ R/importWQP.R | 2 ++ 2 files changed, 4 insertions(+) diff --git a/NEWS b/NEWS index 0edaf0da..5703ef73 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ dataRetrieval 2.7.15 =================== * Update warning about upcoming changes to USGS water quality data. +* Added date sorting to all WQP functions. + dataRetrieval 2.7.14 =================== diff --git a/R/importWQP.R b/R/importWQP.R index 252cce65..fb2a48bd 100644 --- a/R/importWQP.R +++ b/R/importWQP.R @@ -269,6 +269,8 @@ importWQP <- function(obs_url, zip = TRUE, tz = "UTC", retval$ActivityStartDateTime <- lubridate::fast_strptime(retval$ActivityStartDateTime, "%Y-%m-%d %H:%M:%S") + 60 * 60 * retval$timeZoneStart attr(retval$ActivityStartDateTime, "tzone") <- tz + # if we're going to sort, here's where we'd do it: + retval <- retval[order(retval$ActivityStartDateTime),] } if (all(c("ActivityEndDate", "ActivityEndTime/Time") %in% names(retval))) { From a253bde9b8b75b31999f204176cebde6a32e7f52 Mon Sep 17 00:00:00 2001 From: Laura A DeCicco Date: Fri, 9 Feb 2024 15:06:17 -0600 Subject: [PATCH 4/7] move files to Temp --- .Rbuildignore | 1 + .gitignore | 1 + 2 files changed, 2 insertions(+) diff --git a/.Rbuildignore b/.Rbuildignore index 668524ec..9b1aec69 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1,4 +1,5 @@ ^Meta$ +^Temp$ ^doc$ .Rhistory vignettes/figures diff --git a/.gitignore b/.gitignore index 25590090..f5369ab4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ docs .DS_Store /doc/ /Meta/ +/Temp/ From b0d419c4166adf0d8bdff88cb9e0a7fb6524f1e2 Mon Sep 17 00:00:00 2001 From: Laura A DeCicco Date: Fri, 9 Feb 2024 15:31:46 -0600 Subject: [PATCH 5/7] Don't include the status page on a CRAN release, so that we have total control of the message (otherwise we'd need to resubmit to CRAN each time). --- .Rbuildignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.Rbuildignore b/.Rbuildignore index 9b1aec69..0aa1a16d 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -89,5 +89,6 @@ vignettes/github_code.png vignettes/save_pat.png vignettes/more_tests.png vignettes/stylesSlides.css +vignettes/Status.Rmd ^ci$ ^public$ \ No newline at end of file From d2ac608d65c91436cb4b0f0e4287851d09d1d7cf Mon Sep 17 00:00:00 2001 From: Laura A DeCicco Date: Fri, 9 Feb 2024 15:38:07 -0600 Subject: [PATCH 6/7] wayward http --- R/citations.R | 2 +- R/dataRetrievals-package.R | 2 +- vignettes/dataRetrieval.Rmd | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/R/citations.R b/R/citations.R index 1eba2140..aedbd5ba 100644 --- a/R/citations.R +++ b/R/citations.R @@ -28,7 +28,7 @@ create_NWIS_bib <- function(x){ format(attr(x, "queryTime"), "%b %d, %Y"), ", at ", attr(x, "url"), - ", http://dx.doi.org/10.5066/F7P55KJN") + ", https://dx.doi.org/10.5066/F7P55KJN") ref <- utils::bibentry( bibtype = "Manual", diff --git a/R/dataRetrievals-package.R b/R/dataRetrievals-package.R index 1e3b95eb..96ae93f5 100644 --- a/R/dataRetrievals-package.R +++ b/R/dataRetrievals-package.R @@ -21,7 +21,7 @@ Extended Documentation: https://doi-usgs.github.io/dataRetrieval") #' #' Retrieval functions for USGS and EPA hydrologic and water quality data. #' -#' Please see \url{https://pubs.usgs.gov/publication/tm4A10} for more information. +#' Please see \url{https://doi-usgs.github.io/dataRetrieval/} for more information. #' #' @name dataRetrieval #' @aliases dataRetrieval-package diff --git a/vignettes/dataRetrieval.Rmd b/vignettes/dataRetrieval.Rmd index f44943a9..99428a2e 100644 --- a/vignettes/dataRetrieval.Rmd +++ b/vignettes/dataRetrieval.Rmd @@ -724,7 +724,7 @@ citation(package = "dataRetrieval") The requested format is: -U.S. Geological Survey, 2023, National Water Information System data available on the World Wide Web (USGS Water Data for the Nation), accessed [April 26, 2023], at http://waterdata.usgs.gov/nwis/. http://dx.doi.org/10.5066/F7P55KJN +U.S. Geological Survey, 2023, National Water Information System data available on the World Wide Web (USGS Water Data for the Nation), accessed [April 26, 2023], at https://waterdata.usgs.gov/nwis/. https://dx.doi.org/10.5066/F7P55KJN This can be created from the attributes of the data using the `create_NWIS_bib` function: @@ -770,7 +770,7 @@ U.S. Geological Survey, 2024, National Water Information System data available on the World Wide Web (USGS Water Data for the Nation), accessed Feb 09, 2024, at https://waterservices.usgs.gov/nwis/dv/?site=09010500&format=waterml,1.1&ParameterCd=00060&StatCd=00003&startDT=1851-01-01, -http://dx.doi.org/10.5066/F7P55KJN +https://dx.doi.org/10.5066/F7P55KJN A BibTeX entry for LaTeX users is From 9f43467b4c70642d5f61b06124504998da9c9139 Mon Sep 17 00:00:00 2001 From: Laura A DeCicco Date: Fri, 9 Feb 2024 15:41:34 -0600 Subject: [PATCH 7/7] Fixing Rd files --- R/readNWISunit.R | 2 +- man/dataRetrieval.Rd | 2 +- man/readNWISuse.Rd | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/R/readNWISunit.R b/R/readNWISunit.R index 86d1ccd8..c7d03447 100644 --- a/R/readNWISunit.R +++ b/R/readNWISunit.R @@ -642,7 +642,7 @@ readNWISstat <- function(siteNumbers, parameterCd, startDate = "", endDate = "", #' @param stateCd could be character (full name, abbreviation, id), or numeric (id). #' Only one is accepted per query. #' @param countyCd could be character (name, with or without "County", or "ALL"), -#' numeric (id), or code{NULL}, which will +#' numeric (id), or \code{NULL}, which will #' return state or national data depending on the stateCd argument. "ALL" may #' also be supplied, which will return data #' for every county in a state. Can be a vector of counties in the same state. diff --git a/man/dataRetrieval.Rd b/man/dataRetrieval.Rd index 82847847..d3e8d485 100644 --- a/man/dataRetrieval.Rd +++ b/man/dataRetrieval.Rd @@ -21,7 +21,7 @@ LazyLoad: \tab yes\cr \details{ Retrieval functions for USGS and EPA hydrologic and water quality data. -Please see \url{https://pubs.usgs.gov/publication/tm4A10} for more information. +Please see \url{https://doi-usgs.github.io/dataRetrieval/} for more information. } \seealso{ Useful links: diff --git a/man/readNWISuse.Rd b/man/readNWISuse.Rd index 835b1adc..34de5ddd 100644 --- a/man/readNWISuse.Rd +++ b/man/readNWISuse.Rd @@ -18,7 +18,7 @@ readNWISuse( Only one is accepted per query.} \item{countyCd}{could be character (name, with or without "County", or "ALL"), -numeric (id), or code{NULL}, which will +numeric (id), or \code{NULL}, which will return state or national data depending on the stateCd argument. "ALL" may also be supplied, which will return data for every county in a state. Can be a vector of counties in the same state.}