diff --git a/Misc/data_call_2022/00_template.md b/Misc/data_call_2022/00_template.md new file mode 100644 index 00000000..de183d38 --- /dev/null +++ b/Misc/data_call_2022/00_template.md @@ -0,0 +1,66 @@ +----------------------------------------------------------- +# 00 (put the code of the country there) +----------------------------------------------------------- + +## Annex 1 + +### series + +### dataseries + + +### group metrics + + +### individual metrics + +## Annex 2 + +### series + +### dataseries + + +### group metrics + + +### individual metrics + + + +## Annex 3 + +### series + +### dataseries + + +### group metrics + + +### individual metrics + + + +## Annex 4 + + + +## Annex 5 + + + +## Annex 6 + + + +## Annex 7 + + + +## Annex 8 + + + +## Annex 10 + diff --git a/Misc/data_call_2022/AL.md b/Misc/data_call_2022/AL.md new file mode 100644 index 00000000..59f6542a --- /dev/null +++ b/Misc/data_call_2022/AL.md @@ -0,0 +1,28 @@ +----------------------------------------------------------- +# AL +----------------------------------------------------------- +## Annex 1 + + +## Annex 2 + +## Annex 3 + + +## Annex 4 + + + +## Annex 5 + + +## Annex 6 + + +## Annex 7 + +## Annex 8 + + + +## Annex 10 diff --git a/Misc/data_call_2022/BE.md b/Misc/data_call_2022/BE.md new file mode 100644 index 00000000..5969d4f8 --- /dev/null +++ b/Misc/data_call_2022/BE.md @@ -0,0 +1,40 @@ +----------------------------------------------------------- +# BE +----------------------------------------------------------- + +## Annex 1 +### done +* 4 values updated (modified series) +* 81 values integrated (new dataseries) +* 1 and 2 new values integrated (new group metrics) + +## Annex 2 +no data + +## Annex 3 +no data + +## Annex 4 +no data + +## Annex 5 + ### done + * replaced two values (duplicates) + * 1 new value integrated (new rows) + +## Annex 6 +no data + +## Annex 7 +### notes +* there was a window with duplicates but there were none and the created excel file was empty (in the shiny in eel_missvalueequal.base it says "No data available in table") + +### done +* 4 new values inserted (new rows) + +## Annex 8 +no data + + +## Annex 10 +no data \ No newline at end of file diff --git a/Misc/data_call_2022/CZ.md b/Misc/data_call_2022/CZ.md new file mode 100644 index 00000000..a12af26a --- /dev/null +++ b/Misc/data_call_2022/CZ.md @@ -0,0 +1,30 @@ +----------------------------------------------------------- +# CZ +----------------------------------------------------------- + +## Annex 1 + + +## Annex 2 + +## Annex 3 + + +## Annex 4 + + + +## Annex 5 + + +## Annex 6 + + +## Annex 7 + +## Annex 8 + + + +## Annex 10 + diff --git a/Misc/data_call_2022/DE.md b/Misc/data_call_2022/DE.md new file mode 100644 index 00000000..d7c1a0ee --- /dev/null +++ b/Misc/data_call_2022/DE.md @@ -0,0 +1,52 @@ +----------------------------------------------------------- +# DE +----------------------------------------------------------- +## Annex 1 +### to do +* need to provide ser_restocking etc. in series info + +### done +* modified 2 series (ser_methods) +* integrated 11 values in dataseries (removed empty rows in templates) + +## Annex 2 + +### done +* integrated 1 new value (new dataseries) + +## Annex 3 +### done +* integrated 1 new value (new dataseries) + +## Annex 4 +### to do +* not integrated (no data) but there is an update to metadata since one of the data providers has changed. Not sure how to do approach this... + + +## Annex 5 +### to do +* not integrated (no data) but there is an update to metadata since one of the data providers has changed. Not sure how to do approach this... + +## Annex 6 +* not provided by DE / empty sheet. Nothing to report from Germany, so this should be fine. + +## Annex 7 +### to do +* not integrated (no data) but there is an update to metadata since one of the data providers has changed. Not sure how to do approach this... + +## Annex 8 +### to do +* not integrated due to a bug being stuck in the loading screen (added to issues); but needs to be integrated, DE provided an update to data! + + +## Annex 10 +### notes +* In the database there was a wron name for a series in sai_info (DE_Elbe_Eider should have been DE_Eide_Eider). This was changed in the database and also for the related group metrics the series was changed to DE_Eide_Eider. Accordingly, the spreadsheet, as provided by DE, was edited (i.e. the existing series info was changed accordingly and the existing group metrics was changed accordingly). + +### to do +* pre-filled series ending with "HIST" and related group metrics should be deleted. It's not clear to the data provider how these are generated and if they are reliable. +* integrate individual metrics, there was a bug... + +### done +* group metrics: 137 and 1509 new values inserted in the group and metric tables +* deleted groups metrics: done using an sql query (see database_edition_2022.sql): 34 groups metrics with qal_id 22 and 10 gr_comment updated for gr_id in (2323,2334,2167,2222,2189,2200,2211,2178,2233,2244); diff --git a/Misc/data_call_2022/DZ.md b/Misc/data_call_2022/DZ.md new file mode 100644 index 00000000..6740a429 --- /dev/null +++ b/Misc/data_call_2022/DZ.md @@ -0,0 +1,28 @@ +----------------------------------------------------------- +# DZ +----------------------------------------------------------- +## Annex 1 + + +## Annex 2 + +## Annex 3 + + +## Annex 4 + + + +## Annex 5 + + +## Annex 6 + + +## Annex 7 + +## Annex 8 + + + +## Annex 10 \ No newline at end of file diff --git a/Misc/data_call_2022/EE.md b/Misc/data_call_2022/EE.md new file mode 100644 index 00000000..7b8965dd --- /dev/null +++ b/Misc/data_call_2022/EE.md @@ -0,0 +1,29 @@ +----------------------------------------------------------- +# EE +----------------------------------------------------------- + +## Annex 1 + + +## Annex 2 + +## Annex 3 + + +## Annex 4 + + + +## Annex 5 + + +## Annex 6 + + +## Annex 7 + +## Annex 8 + + + +## Annex 10 \ No newline at end of file diff --git a/Misc/data_call_2022/EG.md b/Misc/data_call_2022/EG.md new file mode 100644 index 00000000..f14f6604 --- /dev/null +++ b/Misc/data_call_2022/EG.md @@ -0,0 +1,24 @@ +----------------------------------------------------------- +# EG +----------------------------------------------------------- +## Annex 1 + + +## Annex 2 + +## Annex 3 + + +## Annex 4 + + + +## Annex 5 + + +## Annex 6 + + +## Annex 7 + +## Annex 8 \ No newline at end of file diff --git a/Misc/data_call_2022/ES.md b/Misc/data_call_2022/ES.md new file mode 100644 index 00000000..fdab8098 --- /dev/null +++ b/Misc/data_call_2022/ES.md @@ -0,0 +1,24 @@ +----------------------------------------------------------- +# ES +----------------------------------------------------------- +## Annex 1 + + +## Annex 2 + +## Annex 3 + + +## Annex 4 + + + +## Annex 5 + + +## Annex 6 + + +## Annex 7 + +## Annex 8 \ No newline at end of file diff --git a/Misc/data_call_2022/FI.md b/Misc/data_call_2022/FI.md new file mode 100644 index 00000000..346bbfc8 --- /dev/null +++ b/Misc/data_call_2022/FI.md @@ -0,0 +1,28 @@ +----------------------------------------------------------- +# FI +----------------------------------------------------------- +## Annex 1 + + +## Annex 2 + +## Annex 3 + + +## Annex 4 + + + +## Annex 5 + + +## Annex 6 + + +## Annex 7 + +## Annex 8 + + + +## Annex 10 \ No newline at end of file diff --git a/Misc/data_call_2022/FR.md b/Misc/data_call_2022/FR.md new file mode 100644 index 00000000..bb862cf5 --- /dev/null +++ b/Misc/data_call_2022/FR.md @@ -0,0 +1,76 @@ +----------------------------------------------------------- +# FR +----------------------------------------------------------- +## Annex 1 + +### series + +1 new +4 values updated in the db + +### dataseries + +18 new values inserted in the database +296 values updated in the db + +### group metrics + +0 + +### individuals metrics + + 234355 and 486270 new values inserted in the group and metric tables + +## Annex 2 + +### series + +0 new, 0 modified + +### dataseries + +17 new ; 140 values updated in the db + +### group metrics + +0 + +### individual metrics + + 78542 and 141559 new values inserted in the group and metric tables + +## Annex 3 + +### series + +0 new, 0 modified + +### dataseries + +to be checked by LB (issue with season) + +## Annex 4 + + 121 new values inserted in the database +28 values updated in the db + +## Annex 5 + + 115 new values inserted in the database +9 values updated in the db + +## Annex 6 + +No data + +## Annex 7 + +error 'there is an error' + +## Annex 8 + +No data + +## Annex 10 + +on error \ No newline at end of file diff --git a/Misc/data_call_2022/GB.md b/Misc/data_call_2022/GB.md new file mode 100644 index 00000000..b2295039 --- /dev/null +++ b/Misc/data_call_2022/GB.md @@ -0,0 +1,44 @@ + + + +----------------------------------------------------------- +# GB +----------------------------------------------------------- +## Annex 1 + + +## Annex 2 + +## Annex 3 + + +## Annex 4 +305 new values inserted in the database + +103 values updated in the db + +## Annex 5 +168 new values inserted in the database + +108 values updated in the db + +## Annex 6 +5 new values inserted in the database + +1 values updated in the db + + +## Annex 7 +72 new values inserted in the database + +## Annex 8 +Not relevant to GB. + + +## Annex 10 +* 3 new rows added under sampling info + +* 23 and 283 new values inserted in the group and metric tables +* deleted groups metrics: done using an sql query (see database_edition_2022.sql): 34 groups metrics with qal_id 22 and 24 gr_comment updated for gr_id in (2176,2177,2179,2180,2181,2182,2183,2169,2170,2171,2172,2173,2174,2175,2184,2185,2186,2187,2188,2190,2191,2192,2193,2194); + +Issue with integrating individual data \ No newline at end of file diff --git a/Misc/data_call_2022/GR.md b/Misc/data_call_2022/GR.md new file mode 100644 index 00000000..a345440b --- /dev/null +++ b/Misc/data_call_2022/GR.md @@ -0,0 +1,32 @@ +----------------------------------------------------------- +# GR +----------------------------------------------------------- +## Annex 1 +-- +## Annex 2 +1 update modified series (Step 2.1.2) and 11 new dataseries (Step 2.2.2) +## Annex 3 +31 new values in new data series +3 values in uptaded modified series +1 and 10 values integrate new group metrics +## Annex 4 + +32 new values was added. + +Error: Failed to prepare query: ERROR: column "eel_value" does not exist +LINE 17: eel_value, + ^ +HINT: There is a column named "eel_value" in table "t_eelstock_eel", but it cannot be referenced from this part of the query. + +## Annex 5 +-- + +## Annex 6 +-- + +## Annex 7 +8 new values was added +## Annex 8 +1new value was added + +## Annex 10 \ No newline at end of file diff --git a/Misc/data_call_2022/IE.md b/Misc/data_call_2022/IE.md new file mode 100644 index 00000000..49c57cf3 --- /dev/null +++ b/Misc/data_call_2022/IE.md @@ -0,0 +1,35 @@ +----------------------------------------------------------- +# IE +----------------------------------------------------------- + +## Annex 1 + + +## Annex 2 + +## Annex 3 + + +## Annex 4 +72 new rows added + +## Annex 5 +72 new rows added +540 rows updated + +## Annex 6 +14 new rows added +3 rows updated + +## Annex 7 +14 new rows +2 updated rows +## Annex 8 + + + +## Annex 10 +* 2 new rows added sampling info +* 6 and 31 new group metrics +* deleted groups metrics: done using an sql query (see database_edition_2022.sql): 72 groups metrics with qal_id 22 and 27 gr_comment updated for gr_id in (2195,2196,2197,2198,2199,2201,2202,2203,2204,2205,2206,2207,2208,2209,2210,2212,2213,2214,2215,2216,2217,2218,2219,2220,2221,2260,2261); + diff --git a/Misc/data_call_2022/IT.md b/Misc/data_call_2022/IT.md new file mode 100644 index 00000000..6bc6b60a --- /dev/null +++ b/Misc/data_call_2022/IT.md @@ -0,0 +1,39 @@ +----------------------------------------------------------- +# IT +----------------------------------------------------------- + +## Annex 1 +No new data + +## Annex 2 +No new data + +## Annex 3 +No new data + +## Annex 4 +### done +* new data inserted 28 rows +### TODO +* duplicates error, needs to be reprogrammed + +## Annex 5 +### done +* new data inserted 2 rows +* * some of the new rows are recognized as duplicates (probably NR yet) +### TODO +* duplicates error, needs to be reprogrammed +## Annex 6 +No data +## Annex 7 +### TODO +2 rows +## Annex 8 +### Done +2 rows +## Annex 10 + +###Done : +Deleted old values +--26 rows deleted +--9 rows updated in group metrics with comment all individual metrics qal_id 22 diff --git a/Misc/data_call_2022/readme.md b/Misc/data_call_2022/readme.md index c49dd2c7..0624e3ed 100644 --- a/Misc/data_call_2022/readme.md +++ b/Misc/data_call_2022/readme.md @@ -1,566 +1,35 @@ ------------------------------------------------------------ -# AL ------------------------------------------------------------ -## Annex 1 -## Annex 2 -## Annex 3 -## Annex 4 -## Annex 5 -## Annex 6 - - -## Annex 7 -## Annex 8 -## Annex 10 ------------------------------------------------------------ -# BE ------------------------------------------------------------ -## Annex 1 -### done -* 4 values updated (modified series) -* 81 values integrated (new dataseries) -* 1 and 2 new values integrated (new group metrics) -## Annex 2 -no data -## Annex 3 -no data -## Annex 4 -no data -## Annex 5 - ### done - * replaced two values (duplicates) - * 1 new value integrated (new rows) -## Annex 6 -no data -## Annex 7 -### notes -* there was a window with duplicates but there were none and the created excel file was empty (in the shiny in eel_missvalueequal.base it says "No data available in table") -### done -* 4 new values inserted (new rows) -## Annex 8 -no data -## Annex 10 -no data ------------------------------------------------------------ -# CZ ------------------------------------------------------------ - -## Annex 1 - - -## Annex 2 - -## Annex 3 - - -## Annex 4 - - - -## Annex 5 - - -## Annex 6 - - -## Annex 7 - -## Annex 8 -## Annex 10 - - ------------------------------------------------------------ -# DE ------------------------------------------------------------ -## Annex 1 -### to do -* need to provide ser_restocking etc. in series info - -### done -* modified 2 series (ser_methods) -* integrated 11 values in dataseries (removed empty rows in templates) - -## Annex 2 - -### done -* integrated 1 new value (new dataseries) - -## Annex 3 -### done -* integrated 1 new value (new dataseries) - -## Annex 4 -### to do -* not integrated (no data) but there is an update to metadata since one of the data providers has changed. Not sure how to do approach this... - - -## Annex 5 -### to do -* not integrated (no data) but there is an update to metadata since one of the data providers has changed. Not sure how to do approach this... - -## Annex 6 -* not provided by DE / empty sheet. Nothing to report from Germany, so this should be fine. - -## Annex 7 -### to do -* not integrated (no data) but there is an update to metadata since one of the data providers has changed. Not sure how to do approach this... - -## Annex 8 -### to do -* not integrated due to a bug being stuck in the loading screen (added to issues); but needs to be integrated, DE provided an update to data! - - -## Annex 10 -### notes -* In the database there was a wron name for a series in sai_info (DE_Elbe_Eider should have been DE_Eide_Eider). This was changed in the database and also for the related group metrics the series was changed to DE_Eide_Eider. Accordingly, the spreadsheet, as provided by DE, was edited (i.e. the existing series info was changed accordingly and the existing group metrics was changed accordingly). - -### to do -* pre-filled series ending with "HIST" and related group metrics should be deleted. It's not clear to the data provider how these are generated and if they are reliable. -* integrate individual metrics, there was a bug... - -### done -* group metrics: 137 and 1509 new values inserted in the group and metric tables -* deleted groups metrics: done using an sql query (see database_edition_2022.sql): 34 groups metrics with qal_id 22 and 10 gr_comment updated for gr_id in (2323,2334,2167,2222,2189,2200,2211,2178,2233,2244); ------------------------------------------------------------ -# DK ------------------------------------------------------------ -## Annex 1 -7 series modified -In new dataseries: Error: Failed to fetch row: ERROR: duplicate key value violates unique constraint "c_uk_year_ser_id" -DETAIL: Key (das_year, das_ser_id)=(2016, 39) already exists. - -## Annex 2 - -## Annex 3 - - -## Annex 4 - - - -## Annex 5 - - -## Annex 6 - - -## Annex 7 - -## Annex 8 - - - -## Annex 10 - ------------------------------------------------------------ -# DZ ------------------------------------------------------------ -## Annex 1 - - -## Annex 2 - -## Annex 3 - - -## Annex 4 - - - -## Annex 5 - - -## Annex 6 - - -## Annex 7 - -## Annex 8 - - - -## Annex 10 - ------------------------------------------------------------ -# EE ------------------------------------------------------------ - -## Annex 1 - - -## Annex 2 - -## Annex 3 - - -## Annex 4 - - - -## Annex 5 - - -## Annex 6 - - -## Annex 7 - -## Annex 8 - - - -## Annex 10 - - - - ------------------------------------------------------------ -# EG ------------------------------------------------------------ -## Annex 1 - - -## Annex 2 - -## Annex 3 - - -## Annex 4 - - - -## Annex 5 - - -## Annex 6 - - -## Annex 7 - -## Annex 8 - -## Annex 10 - ------------------------------------------------------------ -# ES ------------------------------------------------------------ -## Annex 1 - - -## Annex 2 - -## Annex 3 - - -## Annex 4 - - - -## Annex 5 - - -## Annex 6 - - -## Annex 7 - -## Annex 8 - - - -## Annex 10 - ------------------------------------------------------------ -# FI ------------------------------------------------------------ -## Annex 1 - - -## Annex 2 - -## Annex 3 - - -## Annex 4 - - - -## Annex 5 - - -## Annex 6 - - -## Annex 7 - -## Annex 8 - - - -## Annex 10 - ------------------------------------------------------------ -# FR ------------------------------------------------------------ -## Annex 1 - -### series - -1 new -4 values updated in the db - -### dataseries - -18 new values inserted in the database -296 values updated in the db - -### group metrics - -0 - -### individuals metrics - - 234355 and 486270 new values inserted in the group and metric tables - -## Annex 2 - -### series - -0 new, 0 modified - -### dataseries - -17 new ; 140 values updated in the db - -### group metrics - -0 - -### individual metrics - - 78542 and 141559 new values inserted in the group and metric tables - -## Annex 3 - -### series - -0 new, 0 modified - -### dataseries - -to be checked by LB (issue with season) - -## Annex 4 - - 121 new values inserted in the database -28 values updated in the db - -## Annex 5 - - 115 new values inserted in the database -9 values updated in the db - -## Annex 6 - -No data - -## Annex 7 - -error 'there is an error' - -## Annex 8 - -No data - -## Annex 10 - -on error - ------------------------------------------------------------ -# GR ------------------------------------------------------------ -## Annex 1 --- -## Annex 2 -1 update modified series (Step 2.1.2) and 11 new dataseries (Step 2.2.2) -## Annex 3 -31 new values in new data series -3 values in uptaded modified series -1 and 10 values integrate new group metrics -## Annex 4 - -32 new values was added. - -Error: Failed to prepare query: ERROR: column "eel_value" does not exist -LINE 17: eel_value, - ^ -HINT: There is a column named "eel_value" in table "t_eelstock_eel", but it cannot be referenced from this part of the query. - -## Annex 5 --- - -## Annex 6 --- - -## Annex 7 -8 new values was added -## Annex 8 -1new value was added - -## Annex 10 --- ------------------------------------------------------------ -# HR ------------------------------------------------------------ - -Croatia do we have anything ? - -# GB ------------------------------------------------------------ -## Annex 1 - - -## Annex 2 - -## Annex 3 - - -## Annex 4 -305 new values inserted in the database - -103 values updated in the db - -## Annex 5 -168 new values inserted in the database - -108 values updated in the db - -## Annex 6 -5 new values inserted in the database - -1 values updated in the db - - -## Annex 7 -72 new values inserted in the database - -## Annex 8 -Not relevant to GB. - - -## Annex 10 -* 3 new rows added under sampling info - -* 23 and 283 new values inserted in the group and metric tables -* deleted groups metrics: done using an sql query (see database_edition_2022.sql): 34 groups metrics with qal_id 22 and 24 gr_comment updated for gr_id in (2176,2177,2179,2180,2181,2182,2183,2169,2170,2171,2172,2173,2174,2175,2184,2185,2186,2187,2188,2190,2191,2192,2193,2194); - -Issue with integrating individual data - ------------------------------------------------------------ -# IE ------------------------------------------------------------ - -## Annex 1 - - -## Annex 2 - -## Annex 3 - - -## Annex 4 -72 new rows added - -## Annex 5 -72 new rows added -540 rows updated - -## Annex 6 -14 new rows added -3 rows updated - -## Annex 7 -14 new rows -2 updated rows -## Annex 8 - - - -## Annex 10 -* 2 new rows added sampling info -* 6 and 31 new group metrics -* deleted groups metrics: done using an sql query (see database_edition_2022.sql): 72 groups metrics with qal_id 22 and 27 gr_comment updated for gr_id in (2195,2196,2197,2198,2199,2201,2202,2203,2204,2205,2206,2207,2208,2209,2210,2212,2213,2214,2215,2216,2217,2218,2219,2220,2221,2260,2261); - - - ------------------------------------------------------------ -# IT ------------------------------------------------------------ - -## Annex 1 -No new data - -## Annex 2 -No new data - -## Annex 3 -No new data - -## Annex 4 -### done -* new data inserted 28 rows -### TODO -* duplicates error, needs to be reprogrammed - -## Annex 5 -### done -* new data inserted 2 rows -* * some of the new rows are recognized as duplicates (probably NR yet) -### TODO -* duplicates error, needs to be reprogrammed -## Annex 6 -No data -## Annex 7 -### TODO -2 rows -## Annex 8 -### Done -2 rows -## Annex 10 -### TODO -Shiny delete the old data ----------------------------------------------------------- diff --git a/R/shiny_data_integration/shiny_di/database_tools.R b/R/shiny_data_integration/shiny_di/database_tools.R index 8e8de67c..9d4d9acd 100644 --- a/R/shiny_data_integration/shiny_di/database_tools.R +++ b/R/shiny_data_integration/shiny_di/database_tools.R @@ -515,7 +515,7 @@ compare_with_database_dataseries <- function(data_from_excel, data_from_base, sh } } - + modified <- dplyr::anti_join(data_from_excel, data_from_base, by = c("das_year", "das_value", "das_comment", "das_effort", "das_ser_id", "das_qal_id") ) @@ -964,7 +964,8 @@ compare_with_database_metric_ind <- function( #' } #' } #' @rdname write_duplicate -write_duplicates <- function(path, qualify_code = 19) { +write_duplicates <- function(path, qualify_code = 22) { + duplicates2 <- read_excel(path = path, sheet = 1, skip = 1) # Initial checks ---------------------------------------------------------------------------------- @@ -1028,8 +1029,8 @@ write_duplicates <- function(path, qualify_code = 19) { # this will perform the reverse operation if error in query 1 or 2 # sqldf will handle this one as it is a several liners - query0_reverse <- paste0("update datawg.t_eelstock_eel set (eel_qal_id,eel_comment)=(", - replaced$eel_qal_id.base , ",'", replaced$eel_comment.base, "') where eel_id=", replaced$eel_id,";") +# query0_reverse <- paste0("update datawg.t_eelstock_eel set (eel_qal_id,eel_comment)=(", +# replaced$eel_qal_id.base , ",'", replaced$eel_comment.base, "') where eel_id=", replaced$eel_id,";") # this query will be run later cause we don't want it to run if the other fail @@ -1091,25 +1092,14 @@ write_duplicates <- function(path, qualify_code = 19) { # this query will be run to rollback when query2 crashes #records in t_eel_stockeel_percent are deleted automatically by cascade - query1_reverse <- str_c("delete from datawg.t_eelstock_eel", - " where eel_datelastupdate = current_date", - " and eel_cou_code='",cou_code,"'", - " and eel_datasource='",the_eel_datasource ,"';") +# query1_reverse <- str_c("delete from datawg.t_eelstock_eel", +# " where eel_datelastupdate = current_date", +# " and eel_cou_code='",cou_code,"'", +# " and eel_datasource='",the_eel_datasource ,"';") - } else { - showNotification( - "You don't have any lines in sheet duplicated marked with true in column 'keep new values?', have you forgotten to indicate which lines you want to add in the database ?", - duration = 20, - type = "warning" - ) - query0 <- "" - query0_reverse <- "" - query1 <- "" - query1_reverse <- "" - - } + } # Values not chosen, but we store them in the database -------------------------------------------- @@ -1132,7 +1122,8 @@ write_duplicates <- function(path, qualify_code = 19) { not_replaced$eel_qal_comment.xls <- iconv(not_replaced$eel_qal_comment.xls,"UTF8") not_replaced$eel_comment.xls <- iconv(not_replaced$eel_comment.xls,"UTF8") - + #browser() + colnames(not_replaced) <- gsub(".xls","",colnames(not_replaced)) query2 <- str_c( "insert into datawg.t_eelstock_eel ( eel_typ_id, eel_year, @@ -1159,7 +1150,7 @@ write_duplicates <- function(path, qualify_code = 19) { eel_qal_id, eel_qal_comment, eel_datasource, - eel_comment from not_replaced_temp_",cou_code," returning eel_id;") + eel_comment from not_replaced_temp_",cou_code) query2bis <- str_c( "insert into datawg.t_eelstock_eel_percent ( percent_id, perc_f, @@ -1172,103 +1163,83 @@ write_duplicates <- function(path, qualify_code = 19) { perc_c, perc_mo from not_replaced_temp_",cou_code,";") - } else { - - query2 <- "" - } + } + #browser() - # Inserting temporary tables - # running this with more than one sesssion might lead to crash conn <- poolCheckout(pool) - dbExecute(conn,str_c("drop table if exists not_replaced_temp_",cou_code) ) - dbWriteTable(conn,str_c("not_replaced_temp_", tolower(cou_code)),not_replaced,temporary=TRUE,row.names=FALSE ) - dbExecute(conn,str_c("drop table if exists replaced_temp_",cou_code) ) - dbWriteTable(conn, str_c("replaced_temp_", tolower(cou_code)), replaced,temporary=TRUE,row.names=FALSE ) - - - # Insertion of the three queries ---------------------------------------------------------------- - - # if fails replaces the message with this trycatch ! I've tried many ways with - # sqldf but trycatch failed to catch the error Hence the use of DBI - # - - message <- NULL - # First step, replace values in the database -------------------------------------------------- - - #sqldf(query0) - nr0 <- tryCatch({ - dbExecute(conn, query0) + tryCatch({ + dbBegin(conn) + dbExecute(conn,str_c("drop table if exists not_replaced_temp_",cou_code) ) + dbWriteTable(conn,str_c("not_replaced_temp_", tolower(cou_code)),not_replaced, temporary=TRUE, row.names=FALSE ) + dbExecute(conn,str_c("drop table if exists replaced_temp_",cou_code) ) + dbWriteTable(conn, str_c("replaced_temp_", tolower(cou_code)), replaced, temporary=TRUE, row.names=FALSE ) + # First step, replace values in the database -------------------------------------------------- + nr0 <-dbExecute(conn, query0) # this will be the same count as inserted nr1 + # Second step insert replaced ------------------------------------------------------------------ + if (nrow(replaced)>0){ + nr1 <- dbExecute(conn, query1) + if (sum(startsWith(names(replaced),"perc_"))>0) { #we have to update also t_eelsock_eel_perc + nr1bis <- dbExecute(conn,query1bis) + } else { + nr1bis <- 0 + } + } else { + showNotification( + "You don't have any lines in sheet duplicated marked with true in column 'keep new values?', have you forgotten to indicate which lines you want to add in the database ?", + duration = 20, + type = "warning" + ) + nr1 <- 0 + nr1bis <- 0 + } + # Third step insert not replaced values into the database with qal id 22----------------------------------------- + if (nrow(not_replaced)>0){ + nr2 <- dbExecute(conn, query2) + if (sum(startsWith(names(not_replaced),"perc_"))>0) { #we have to update also t_eelsock_eel_perc + nr2bis <- dbExecute(conn,query2bis) # nrow not replaced + } else { + nr2bis <- 0 + } + } else { + showNotification( + "All values had FALSE in 'keep new values', no new value inserted in the database", + duration = 20, + type = "warning" + ) + nr2 <-0 + nr2bis <- 0 + } + dbExecute(conn,str_c("drop table if exists not_replaced_temp_",cou_code) ) + dbExecute(conn,str_c("drop table if exists replaced_temp_",cou_code) ) + dbCommit(conn) # if goes to there commit + message <- sprintf( + "For duplicates %s values replaced in the t_eelstock_ eel table (values from current datacall stored with code eel_qal_id %s)\n, + %s values not replaced (values from current datacall stored with code eel_qal_id %s), + ", nr1, qualify_code, nr2, nr2bis, qualify_code) + if (nr1bis+nr2bis>0) { + message <- c(message, sprintf("\n In addition, %s values replaced in the t_eelstock_eel_percent (old values kept with code eel_qal_id=%s)\n, + %s values not replaced for table t_eelstock_eel_percent (values from current datacall stored with code eel_qal_id %s)", + nr1bis, qualify_code, nr2bis, nr2bis, qualify_code)) + } + }, error = function(e) { message <<- e - cat("step1 message :") - print(message) - }, finally = { - #poolReturn(conn) - - }) + cat(" message :") + print(message) + dbRollback(conn) + }, warning = function(e) { + message <<- e + cat(" message :") + print(message) + dbRollback(conn) + }, + finally = { + }) - # Second step insert replaced ------------------------------------------------------------------ - if (is.null(message)) { - #conn <- poolCheckout(pool) - nr1 <- tryCatch({ - if (nrow(replaced)>0){ - replaced$eel_id_new <- dbGetQuery(conn, query1)[,1] - if (sum(startsWith(names(replaced),"perc_"))>0) { #we have to update also t_eelsock_eel_perc - dbExecute(conn,str_c("drop table if exists replaced_temp_",cou_code) ) - dbWriteTable(conn, str_c("replaced_temp_", tolower(cou_code)), replaced,temporary=TRUE,row.names=FALSE ) - dbExecute(conn,query1bis) - } - } - nrow(replaced) - }, error = function(e) { - message <<- e - dbGetQuery(conn, query0_reverse) # perform reverse operation - cat("step2 message :") - print(message) - }, finally = { - #poolReturn(conn) - dbExecute(conn, str_c( "drop table if exists replaced_temp_", cou_code)) - }) - } - # Third step insert not replaced values into the database ----------------------------------------- - - - if (is.null(message)){ # the previous operation had no error - #conn <- poolCheckout(pool) - nr2 <- tryCatch({ - if (nrow(not_replaced)>0){ - not_replaced$eel_id_new <- dbGetQuery(conn, query2)[,1] - if (sum(startsWith(names(not_replaced),"perc_"))>0) { #we have to update also t_eelsock_eel_perc - dbExecute(conn,str_c("drop table if exists not_replaced_temp_",cou_code) ) - dbWriteTable(conn,str_c("not_replaced_temp_", tolower(cou_code)),not_replaced,temporary=TRUE,row.names=FALSE ) - dbExecute(conn,query2bis) - } - } - nrow(not_replaced) - }, error = function(e) { - message <<- e - cat("step3 message :") - print(message) - dbExecute(conn, query1_reverse) # this is not surrounded by trycatch, pray it does not fail .... - dbGetQuery(conn,query0_reverse) # perform reverse operation - }, finally = { - #poolReturn(conn) - dbExecute(conn, str_c( "drop table if exists not_replaced_temp_", cou_code)) - }) - - } else { - dbExecute(conn, str_c( "drop table if exists not_replaced_temp_", cou_code)) - } - if (is.null(message)){ - message <- sprintf("For duplicates %s values replaced in the database (old values kept with code eel_qal_id=%s)\n, - %s values not replaced (values from current datacall stored with code eel_qal_id %s)", - nr1, qualify_code, nr2, qualify_code) - } - poolReturn(conn) return(list(message = message, cou_code = cou_code)) } @@ -1734,8 +1705,8 @@ write_new_sampling <- function(path) { message <- NULL (nr <- tryCatch({ dbExecute(conn, query) - query <- "SELECT * FROM datawg.t_samplinginfo_sai" - t_samplinginfo_sai <<- dbGetQuery(conn, sqlInterpolate(ANSI(), query)) + query <- "SELECT * FROM datawg.t_samplinginfo_sai" + t_samplinginfo_sai <<- dbGetQuery(conn, sqlInterpolate(ANSI(), query)) }, error = function(e) { message <<- e }, finally = { @@ -2108,7 +2079,7 @@ write_new_group_metrics <- function(path, type="series") { fk <- "grsa_sai_id" } new <- read_excel(path = path, sheet = 1, skip = 1) %>% - mutate(gr_number=as.numeric(gr_number)) + mutate(gr_number=as.numeric(gr_number)) if (nrow(new) == 0){ message <- "nothing to import" cou_code <- "" @@ -2295,10 +2266,10 @@ write_new_individual_metrics <- function(path, type="series"){ } else if (any(is.na(new[,fk]))){ wrong <- as.character(unique(new[is.na(new[,fk]),"ser_nameshort"])) if (all(is.na(new[,fk]))){ - cou_code <- "" - # here stop otherwise when sending wrong country name "" crashes when writing log - stop(paste("All missing",fk,"have you forgotten to rerun step 1 after integrating new series or sampling_info ? Series",wrong)) - } else { + cou_code <- "" + # here stop otherwise when sending wrong country name "" crashes when writing log + stop(paste("All missing",fk,"have you forgotten to rerun step 1 after integrating new series or sampling_info ? Series",wrong)) + } else { if (type=="series"){ cou_code = dbGetQuery(conn,paste0("SELECT ser_cou_code FROM datawg.t_series_ser WHERE ser_id='", new$fiser_ser_id[!is.na(new$fiser_ser_id)][1],"';"))$ser_cou_code @@ -2378,7 +2349,7 @@ write_new_individual_metrics <- function(path, type="series"){ dbExecute(conn,"drop table if exists indiv_metrics_tmp") dbCommit(conn) } , warning = function(e) { - shinybusy::remove_modal_spinner() + shinybusy::remove_modal_spinner() message <<- e dbRollback(conn) }, error = function(e) { diff --git a/R/shiny_data_integration/shiny_di/importdcfstep0.R b/R/shiny_data_integration/shiny_di/importdcfstep0.R index c86204e7..fe673e39 100644 --- a/R/shiny_data_integration/shiny_di/importdcfstep0.R +++ b/R/shiny_data_integration/shiny_di/importdcfstep0.R @@ -10,7 +10,7 @@ importdcfstep0UI <- function(id){ ns <- NS(id) tagList(useShinyjs(), - h2("Datacall DCF data - quality - biometry integration"), + h2("Datacall DCF and other sampling data - quality - biometry integration"), h2("step 0 : Data check"), tabsetPanel(tabPanel("MAIN", fluidRow( diff --git a/R/shiny_data_integration/shiny_di/loading_functions.R b/R/shiny_data_integration/shiny_di/loading_functions.R index 42ce29e3..24370fd9 100644 --- a/R/shiny_data_integration/shiny_di/loading_functions.R +++ b/R/shiny_data_integration/shiny_di/loading_functions.R @@ -3031,7 +3031,7 @@ load_dcf<-function(path,datasource){ mutate(fi_year=as.integer(fi_year)) } - + if (grepl("group", sheet)) { data_xls$gr_dts_datasource <- datasource data_xls$meg_dts_datasource <- datasource