diff --git a/DESCRIPTION b/DESCRIPTION index 39bf0dc..02b5d44 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: ContDataQC Title: Quality Control (QC) of Continous Monitoring Data -Version: 2.0.7.9021 +Version: 2.0.7.9022 Authors@R: c( person("Erik W", "Leppo", email="Erik.Leppo@tetratech.com",role=c("aut","cre")), person("Ann","Roseberry Lincoln", role="ctb"), diff --git a/NEWS b/NEWS index 5e53bc3..c5be4bf 100644 --- a/NEWS +++ b/NEWS @@ -3,11 +3,28 @@ NEWS-ContDataQC - #> Last Update: 2023-09-14 17:11:42.524289 + #> Last Update: 2023-09-18 08:25:37.45011 # Version History -## v2.0.7.9019 +## v2.0.7.9022 + +2023-09-18 + +- fix: Change stats N from total records to non-NA records, Issue \#140 + - length(x) to sum(!is.na(x)) + - SumStats.updated.R + - fun.Stats.R + - fun.Stats.File.R + - fun.PeriodStats.R + +## v2.0.7.9021 + +2023-09-15 + +- refactor: Add text for tab references instead of links + +## v2.0.7.9020 2023-09-14 diff --git a/NEWS.md b/NEWS.md index 5e53bc3..c5be4bf 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,11 +3,28 @@ NEWS-ContDataQC - #> Last Update: 2023-09-14 17:11:42.524289 + #> Last Update: 2023-09-18 08:25:37.45011 # Version History -## v2.0.7.9019 +## v2.0.7.9022 + +2023-09-18 + +- fix: Change stats N from total records to non-NA records, Issue \#140 + - length(x) to sum(!is.na(x)) + - SumStats.updated.R + - fun.Stats.R + - fun.Stats.File.R + - fun.PeriodStats.R + +## v2.0.7.9021 + +2023-09-15 + +- refactor: Add text for tab references instead of links + +## v2.0.7.9020 2023-09-14 diff --git a/NEWS.rmd b/NEWS.rmd index 35a1c0c..80e8986 100644 --- a/NEWS.rmd +++ b/NEWS.rmd @@ -21,6 +21,16 @@ cat(paste0("Last Update: ",Sys.time())) # Version History +## v2.0.7.9022 +2023-09-18 + +* fix: Change stats N from total records to non-NA records, Issue #140 + + length(x) to sum(!is.na(x)) + + SumStats.updated.R + + fun.Stats.R + + fun.Stats.File.R + + fun.PeriodStats.R + ## v2.0.7.9021 2023-09-15 diff --git a/R/SumStats.updated.R b/R/SumStats.updated.R index e603587..3762d49 100644 --- a/R/SumStats.updated.R +++ b/R/SumStats.updated.R @@ -82,7 +82,7 @@ SumStats.updated <- function(fun.myFile {##FUN.fun.Stats.START # 00. Debugging Variables#### boo_DEBUG <- FALSE - if(boo_DEBUG==TRUE) {##IF.boo_DEBUG.START + if (boo_DEBUG == TRUE) { fun.myFile <- "DATA_test2_Aw_20130101_20141231.csv" fun.myDir.import <- file.path(".","data-raw") fun.myParam.Name <- c("Water.Temp.C", "Sensor.Depth.ft") @@ -92,13 +92,13 @@ SumStats.updated <- function(fun.myFile fun.myConfig <- "" # Load environment #ContData.env <- new.env(parent = emptyenv()) # in config.R - source(file.path(getwd(),"R","config.R"), local=TRUE) + source(file.path(getwd(), "R", "config.R"), local = TRUE) # might have to load manually }##IF.boo_DEBUG.END # 0a.0. Load environment # config file load, 20170517 - if (fun.myConfig!="") {##IF.fun.myConfig.START + if (fun.myConfig != "") { config.load(fun.myConfig) }##IF.fun.myConfig.START @@ -113,7 +113,7 @@ SumStats.updated <- function(fun.myFile # 2.0. Load Data#### # 2.1. Error Checking, make sure file exists - if(fun.myFile %in% list.files(path=fun.myDir.import)==FALSE) {##IF.file.START + if (fun.myFile %in% list.files(path = fun.myDir.import) == FALSE) { # myMsg <- paste0("Provided file (" ,fun.myFile @@ -125,12 +125,12 @@ SumStats.updated <- function(fun.myFile }##IF.file.END # 2.2. Load File df.load <- utils::read.csv(file.path(fun.myDir.import, fun.myFile) - ,as.is=TRUE,na.strings=c("","NA")) + ,as.is = TRUE, na.strings = c("", "NA")) # 2.3. Error Checking, data field names param.len <- length(fun.myParam.Name) myNames2Match <- c(fun.myParam.Name, fun.myDateTime.Name) #myNames2Match %in% names(df.load) - if(sum(myNames2Match %in% names(df.load))!= (param.len + 1)){##IF.match.START + if (sum(myNames2Match %in% names(df.load)) != (param.len + 1)) { # find non match Names.NonMatch <- myNames2Match[is.na(match(myNames2Match, names(df.load)))] myMsg <- paste0("Provided data file (" @@ -147,12 +147,12 @@ SumStats.updated <- function(fun.myFile param.len <- length(fun.myParam.Name) # Loop, Stats #### - if(boo_DEBUG==TRUE) {##IF.boo_DEBUG.START + if (boo_DEBUG == TRUE) { i <- fun.myParam.Name[1] }##IF.boo_DEBUG.END # 20181114, added for 2nd parameter df.list <- list() - for (i in fun.myParam.Name){##FOR.i.START + for (i in fun.myParam.Name) { # i.num <- match(i, fun.myParam.Name) print(paste0("WORKING on parameter (", i.num,"/",param.len,"); ", i)) @@ -170,16 +170,16 @@ SumStats.updated <- function(fun.myFile # # QC.1. Define parameter flag field ## If flag parameter names is different from config then it won't be found - myParam.Name.Flag <- paste(ContData.env$myName.Flag, i, sep=".") + myParam.Name.Flag <- paste(ContData.env$myName.Flag, i, sep = ".") # QC.2. Modify columns to keep (see 3.2.) based on presence of "flag" field ## give user feedback - if (myParam.Name.Flag %in% names(df.load)) {##IF.flagINnames.START + if (myParam.Name.Flag %in% names(df.load)) { # QC.2.1. Flag field present in data myCol <- c(fun.myDateTime.Name, i, myParam.Name.Flag) # QC.2.1.1. Convert "Fails" to NA where appropriate - if (ContData.env$myStats.Fails.Exclude == TRUE) {##IF.Fails.START + if (ContData.env$myStats.Fails.Exclude == TRUE) { # find Fails - myFails <- df.load[,myParam.Name.Flag]==ContData.env$myFlagVal.Fail + myFails <- df.load[, myParam.Name.Flag] == ContData.env$myFlagVal.Fail myFails.Num <- sum(myFails) # convert to NA df.load[myFails, i] <- NA @@ -217,41 +217,43 @@ included based on user's config file." # 4. Daily Stats for data#### # Calculate daily mean, max, min, range, sd, n # 4.1. Define FUNCTION for use with summaryBy - myQ <- c(0.01,0.05,0.10,0.25,0.50,0.75,0.90,0.95,0.99) + myQ <- c(0.01, 0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95, 0.99) myFUN.Names <- c("mean" - ,"median" - ,"min" - ,"max" - ,"range" - ,"sd" - ,"var" - ,"cv" - ,"n" - ,paste("q",formatC(100*myQ,width=2,flag="0"),sep="")) + , "median" + , "min" + , "max" + , "range" + , "sd" + , "var" + , "cv" + , "n" + ,paste("q" + , formatC(100 * myQ, width = 2, flag = "0") + , sep = "")) # - myFUN.sumBy <- function(x, ...){##FUN.myFUN.sumBy.START - c(mean=mean(x,na.rm=TRUE) - ,median=stats::median(x,na.rm=TRUE) - ,min=min(x,na.rm=TRUE) - ,max=max(x,na.rm=TRUE) - ,range=max(x,na.rm=TRUE)-min(x,na.rm=TRUE) - ,sd=stats::sd(x,na.rm=TRUE) - ,var=stats::var(x,na.rm=TRUE) - ,cv=stats::sd(x,na.rm=TRUE)/mean(x,na.rm=TRUE) - ,n=length(x) - ,q=stats::quantile(x,probs=myQ,na.rm=TRUE) + myFUN.sumBy <- function(x, ...){ + c(mean = mean(x ,na.rm = TRUE) + , median = stats::median(x, na.rm = TRUE) + , min = min(x, na.rm = TRUE) + , max = max(x, na.rm = TRUE) + , range = max(x, na.rm = TRUE) - min(x, na.rm = TRUE) + , sd = stats::sd(x, na.rm = TRUE) + , var = stats::var(x, na.rm = TRUE) + , cv = stats::sd(x, na.rm = TRUE) / mean(x, na.rm = TRUE) + , n = sum(!is.na(x)) + , q = stats::quantile(x, probs = myQ, na.rm = TRUE) ) }##FUN.myFUN.sumBy.END # 4.2. Rename data column (summaryBy doesn't like variables) names(df.param)[match(i,names(df.param))] <- "x" # 4.2. Summary df.summary <- doBy::summaryBy(x ~ Date - , data=df.param - , FUN=myFUN.sumBy - , na.rm=TRUE - , var.names=i) + , data = df.param + , FUN = myFUN.sumBy + , na.rm = TRUE + , var.names = i) - new.list <- list(df=df.summary) + new.list <- list(df = df.summary) names(new.list) <- i df.list <- c(df.list,new.list) }##FOR.i.END diff --git a/R/fun.PeriodStats.R b/R/fun.PeriodStats.R index 58923f4..6cba6cd 100644 --- a/R/fun.PeriodStats.R +++ b/R/fun.PeriodStats.R @@ -366,7 +366,7 @@ included based on user's config file." ,sd = stats::sd(x, na.rm = TRUE) ,var = stats::var(x, na.rm = TRUE) ,cv = stats::sd(x, na.rm = TRUE) / mean(x, na.rm = TRUE) - ,n = length(x) + ,n = sum(!is.na(x)) ,q = stats::quantile(x, probs = myQ, na.rm = TRUE) ) }##FUN.myFUN.sumBy.END diff --git a/R/fun.Stats.File.R b/R/fun.Stats.File.R index 310597d..b7cfe26 100644 --- a/R/fun.Stats.File.R +++ b/R/fun.Stats.File.R @@ -439,7 +439,7 @@ fun.Stats.File <- function(fun.myFile ,sd=stats::sd(x,na.rm=TRUE) ,var=stats::var(x,na.rm=TRUE) ,cv=stats::sd(x,na.rm=TRUE)/mean(x,na.rm=TRUE) - ,n=length(x) + ,n=sum(!is.na(x)) ,q=stats::quantile(x,probs=myQ,na.rm=TRUE) ) }##FUN.myFUN.sumBy.END diff --git a/R/fun.Stats.R b/R/fun.Stats.R index f706783..6bac629 100644 --- a/R/fun.Stats.R +++ b/R/fun.Stats.R @@ -463,7 +463,7 @@ fun.Stats <- function(fun.myData.SiteID ,sd=stats::sd(x,na.rm=TRUE) ,var=stats::var(x,na.rm=TRUE) ,cv=stats::sd(x,na.rm=TRUE)/mean(x,na.rm=TRUE) - ,n=length(x) + ,n=sum(!is.na(x)) ,q=stats::quantile(x,probs=myQ,na.rm=TRUE) ) }##FUN.myFUN.sumBy.END diff --git a/inst/shiny-examples/ContDataQC/ui.R b/inst/shiny-examples/ContDataQC/ui.R index 32d6392..5feaae3 100644 --- a/inst/shiny-examples/ContDataQC/ui.R +++ b/inst/shiny-examples/ContDataQC/ui.R @@ -25,7 +25,7 @@ tab_5b_Console <- source("external/tab_5b_Console.R", local = TRUE)$value shinyUI( # VERSION, 1, current [non-EPA] ---- - navbarPage("Continuous data QC, summary, and statistics - v2.0.7.9021", + navbarPage("Continuous data QC, summary, and statistics - v2.0.7.9022", theme = shinytheme("spacelab") ,tab_1_About() #,tab_1a_Overview()