v2.0.7.9022

* fix: Change stats N from total records to non-NA records, Issue #140 + length(x) to sum(!is.na(x)) + SumStats.updated.R + fun.Stats.R + fun.Stats.File.R + fun.PeriodStats.R
leppott · Sep 18, 2023 · 5d6c442 · 5d6c442
1 parent 61d0de2
commit 5d6c442
Show file tree

Hide file tree

Showing 9 changed files with 93 additions and 47 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: ContDataQC
 Title: Quality Control (QC) of Continous Monitoring Data
-Version: 2.0.7.9021
+Version: 2.0.7.9022
 Authors@R: c(
 	person("Erik W", "Leppo", email="[email protected]",role=c("aut","cre")),
 	person("Ann","Roseberry Lincoln", role="ctb"),

diff --git a/NEWS b/NEWS
@@ -3,11 +3,28 @@ NEWS-ContDataQC
 
 <!-- NEWS.md is generated from NEWS.Rmd. Please edit that file -->
 
-    #> Last Update: 2023-09-14 17:11:42.524289
+    #> Last Update: 2023-09-18 08:25:37.45011
 
 # Version History
 
-## v2.0.7.9019
+## v2.0.7.9022
+
+2023-09-18
+
+- fix: Change stats N from total records to non-NA records, Issue \#140
+  - length(x) to sum(!is.na(x))
+  - SumStats.updated.R
+  - fun.Stats.R
+  - fun.Stats.File.R
+  - fun.PeriodStats.R
+
+## v2.0.7.9021
+
+2023-09-15
+
+- refactor: Add text for tab references instead of links
+
+## v2.0.7.9020
 
 2023-09-14
 

diff --git a/NEWS.md b/NEWS.md
@@ -3,11 +3,28 @@ NEWS-ContDataQC
 
 <!-- NEWS.md is generated from NEWS.Rmd. Please edit that file -->
 
-    #> Last Update: 2023-09-14 17:11:42.524289
+    #> Last Update: 2023-09-18 08:25:37.45011
 
 # Version History
 
-## v2.0.7.9019
+## v2.0.7.9022
+
+2023-09-18
+
+- fix: Change stats N from total records to non-NA records, Issue \#140
+  - length(x) to sum(!is.na(x))
+  - SumStats.updated.R
+  - fun.Stats.R
+  - fun.Stats.File.R
+  - fun.PeriodStats.R
+
+## v2.0.7.9021
+
+2023-09-15
+
+- refactor: Add text for tab references instead of links
+
+## v2.0.7.9020
 
 2023-09-14
 

diff --git a/NEWS.rmd b/NEWS.rmd
@@ -21,6 +21,16 @@ cat(paste0("Last Update: ",Sys.time()))
 
 # Version History
 
+## v2.0.7.9022
+2023-09-18
+
+* fix: Change stats N from total records to non-NA records, Issue #140
+    + length(x) to sum(!is.na(x))
+    + SumStats.updated.R
+    + fun.Stats.R
+    + fun.Stats.File.R
+    + fun.PeriodStats.R
+
 ## v2.0.7.9021
 2023-09-15
 

diff --git a/R/SumStats.updated.R b/R/SumStats.updated.R
@@ -82,7 +82,7 @@ SumStats.updated <- function(fun.myFile
 {##FUN.fun.Stats.START
   # 00. Debugging Variables####
   boo_DEBUG <- FALSE
-  if(boo_DEBUG==TRUE) {##IF.boo_DEBUG.START
+  if (boo_DEBUG == TRUE) {
     fun.myFile <- "DATA_test2_Aw_20130101_20141231.csv"
     fun.myDir.import <- file.path(".","data-raw")
     fun.myParam.Name <- c("Water.Temp.C", "Sensor.Depth.ft")
@@ -92,13 +92,13 @@ SumStats.updated <- function(fun.myFile
     fun.myConfig <- ""
     # Load environment
     #ContData.env <- new.env(parent = emptyenv()) # in config.R
-    source(file.path(getwd(),"R","config.R"), local=TRUE)
+    source(file.path(getwd(), "R", "config.R"), local = TRUE)
     # might have to load manually
   }##IF.boo_DEBUG.END
 
   # 0a.0. Load environment
   # config file load, 20170517
-  if (fun.myConfig!="") {##IF.fun.myConfig.START
+  if (fun.myConfig != "") {
     config.load(fun.myConfig)
   }##IF.fun.myConfig.START
 
@@ -113,7 +113,7 @@ SumStats.updated <- function(fun.myFile
 
   # 2.0. Load Data####
   # 2.1. Error Checking, make sure file exists
-  if(fun.myFile %in% list.files(path=fun.myDir.import)==FALSE) {##IF.file.START
+  if (fun.myFile %in% list.files(path = fun.myDir.import) == FALSE) {
     #
     myMsg <- paste0("Provided file ("
                     ,fun.myFile
@@ -125,12 +125,12 @@ SumStats.updated <- function(fun.myFile
   }##IF.file.END
   # 2.2. Load File
   df.load <- utils::read.csv(file.path(fun.myDir.import, fun.myFile)
-                             ,as.is=TRUE,na.strings=c("","NA"))
+                             ,as.is = TRUE, na.strings = c("", "NA"))
   # 2.3. Error Checking, data field names
   param.len <- length(fun.myParam.Name)
   myNames2Match <- c(fun.myParam.Name, fun.myDateTime.Name)
   #myNames2Match %in% names(df.load)
-  if(sum(myNames2Match %in% names(df.load))!= (param.len + 1)){##IF.match.START
+  if (sum(myNames2Match %in% names(df.load)) != (param.len + 1)) {
     # find non match
     Names.NonMatch <- myNames2Match[is.na(match(myNames2Match, names(df.load)))]
     myMsg <- paste0("Provided data file ("
@@ -147,12 +147,12 @@ SumStats.updated <- function(fun.myFile
   param.len <- length(fun.myParam.Name)
 
   # Loop, Stats ####
-  if(boo_DEBUG==TRUE) {##IF.boo_DEBUG.START
+  if (boo_DEBUG == TRUE) {
     i <- fun.myParam.Name[1]
   }##IF.boo_DEBUG.END
   # 20181114, added for 2nd parameter
   df.list <- list()
-  for (i in fun.myParam.Name){##FOR.i.START
+  for (i in fun.myParam.Name) {
     #
     i.num <- match(i, fun.myParam.Name)
     print(paste0("WORKING on parameter (", i.num,"/",param.len,"); ", i))
@@ -170,16 +170,16 @@ SumStats.updated <- function(fun.myFile
     #
     # QC.1. Define parameter flag field
     ## If flag parameter names is different from config then it won't be found
-    myParam.Name.Flag <- paste(ContData.env$myName.Flag, i, sep=".")
+    myParam.Name.Flag <- paste(ContData.env$myName.Flag, i, sep = ".")
     # QC.2. Modify columns to keep (see 3.2.) based on presence of "flag" field
     ## give user feedback
-    if (myParam.Name.Flag %in% names(df.load)) {##IF.flagINnames.START
+    if (myParam.Name.Flag %in% names(df.load)) {
       # QC.2.1. Flag field present in data
       myCol <- c(fun.myDateTime.Name, i, myParam.Name.Flag)
       # QC.2.1.1. Convert "Fails" to NA where appropriate
-      if (ContData.env$myStats.Fails.Exclude == TRUE) {##IF.Fails.START
+      if (ContData.env$myStats.Fails.Exclude == TRUE) {
         # find Fails
-        myFails <- df.load[,myParam.Name.Flag]==ContData.env$myFlagVal.Fail
+        myFails <- df.load[, myParam.Name.Flag] == ContData.env$myFlagVal.Fail
         myFails.Num <- sum(myFails)
         # convert to NA
         df.load[myFails, i] <- NA
@@ -217,41 +217,43 @@ included based on user's config file."
     # 4. Daily Stats for data####
     # Calculate daily mean, max, min, range, sd, n
     # 4.1. Define FUNCTION for use with summaryBy
-    myQ <- c(0.01,0.05,0.10,0.25,0.50,0.75,0.90,0.95,0.99)
+    myQ <- c(0.01, 0.05, 0.10, 0.25, 0.50, 0.75, 0.90, 0.95, 0.99)
     myFUN.Names <- c("mean"
-                     ,"median"
-                     ,"min"
-                     ,"max"
-                     ,"range"
-                     ,"sd"
-                     ,"var"
-                     ,"cv"
-                     ,"n"
-                     ,paste("q",formatC(100*myQ,width=2,flag="0"),sep=""))
+                     , "median"
+                     , "min"
+                     , "max"
+                     , "range"
+                     , "sd"
+                     , "var"
+                     , "cv"
+                     , "n"
+                     ,paste("q"
+                            , formatC(100 * myQ, width = 2, flag = "0")
+                            , sep = ""))
     #
-    myFUN.sumBy <- function(x, ...){##FUN.myFUN.sumBy.START
-      c(mean=mean(x,na.rm=TRUE)
-        ,median=stats::median(x,na.rm=TRUE)
-        ,min=min(x,na.rm=TRUE)
-        ,max=max(x,na.rm=TRUE)
-        ,range=max(x,na.rm=TRUE)-min(x,na.rm=TRUE)
-        ,sd=stats::sd(x,na.rm=TRUE)
-        ,var=stats::var(x,na.rm=TRUE)
-        ,cv=stats::sd(x,na.rm=TRUE)/mean(x,na.rm=TRUE)
-        ,n=length(x)
-        ,q=stats::quantile(x,probs=myQ,na.rm=TRUE)
+    myFUN.sumBy <- function(x, ...){
+      c(mean = mean(x ,na.rm = TRUE)
+        , median = stats::median(x, na.rm = TRUE)
+        , min = min(x, na.rm = TRUE)
+        , max = max(x, na.rm = TRUE)
+        , range = max(x, na.rm = TRUE) - min(x, na.rm = TRUE)
+        , sd = stats::sd(x, na.rm = TRUE)
+        , var = stats::var(x, na.rm = TRUE)
+        , cv = stats::sd(x, na.rm = TRUE) / mean(x, na.rm = TRUE)
+        , n = sum(!is.na(x))
+        , q = stats::quantile(x, probs = myQ, na.rm = TRUE)
       )
     }##FUN.myFUN.sumBy.END
     # 4.2.  Rename data column (summaryBy doesn't like variables)
     names(df.param)[match(i,names(df.param))] <- "x"
     # 4.2. Summary
     df.summary <- doBy::summaryBy(x ~ Date
-                                  , data=df.param
-                                  , FUN=myFUN.sumBy
-                                  , na.rm=TRUE
-                                  , var.names=i)
+                                  , data = df.param
+                                  , FUN = myFUN.sumBy
+                                  , na.rm = TRUE
+                                  , var.names = i)
 
-    new.list <- list(df=df.summary)
+    new.list <- list(df = df.summary)
     names(new.list) <- i
     df.list <- c(df.list,new.list)
   }##FOR.i.END

diff --git a/R/fun.PeriodStats.R b/R/fun.PeriodStats.R
@@ -366,7 +366,7 @@ included based on user's config file."
         ,sd = stats::sd(x, na.rm = TRUE)
         ,var = stats::var(x, na.rm = TRUE)
         ,cv = stats::sd(x, na.rm = TRUE) / mean(x, na.rm = TRUE)
-        ,n = length(x)
+        ,n = sum(!is.na(x))
         ,q = stats::quantile(x, probs = myQ, na.rm = TRUE)
       )
     }##FUN.myFUN.sumBy.END

diff --git a/R/fun.Stats.File.R b/R/fun.Stats.File.R
@@ -439,7 +439,7 @@ fun.Stats.File <- function(fun.myFile
           ,sd=stats::sd(x,na.rm=TRUE)
           ,var=stats::var(x,na.rm=TRUE)
           ,cv=stats::sd(x,na.rm=TRUE)/mean(x,na.rm=TRUE)
-          ,n=length(x)
+          ,n=sum(!is.na(x))
           ,q=stats::quantile(x,probs=myQ,na.rm=TRUE)
           )
       }##FUN.myFUN.sumBy.END

diff --git a/R/fun.Stats.R b/R/fun.Stats.R
@@ -463,7 +463,7 @@ fun.Stats <- function(fun.myData.SiteID
         ,sd=stats::sd(x,na.rm=TRUE)
         ,var=stats::var(x,na.rm=TRUE)
         ,cv=stats::sd(x,na.rm=TRUE)/mean(x,na.rm=TRUE)
-        ,n=length(x)
+        ,n=sum(!is.na(x))
         ,q=stats::quantile(x,probs=myQ,na.rm=TRUE)
         )
     }##FUN.myFUN.sumBy.END

diff --git a/inst/shiny-examples/ContDataQC/ui.R b/inst/shiny-examples/ContDataQC/ui.R
@@ -25,7 +25,7 @@ tab_5b_Console    <- source("external/tab_5b_Console.R", local = TRUE)$value
 
 shinyUI(
   # VERSION, 1, current [non-EPA] ----
-  navbarPage("Continuous data QC, summary, and statistics - v2.0.7.9021",
+  navbarPage("Continuous data QC, summary, and statistics - v2.0.7.9022",
              theme = shinytheme("spacelab")
              ,tab_1_About()
              #,tab_1a_Overview()