From e61560f1fa38af34464bcf5995ab55362b45ed4f Mon Sep 17 00:00:00 2001 From: Clare2D Date: Thu, 18 Feb 2021 09:25:23 +0100 Subject: [PATCH 01/12] fixed col type test --- 3_run_analysis.R | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/3_run_analysis.R b/3_run_analysis.R index 68da092a..a4825bcd 100644 --- a/3_run_analysis.R +++ b/3_run_analysis.R @@ -1,7 +1,7 @@ # TODO: # Clean up sectors options -# port_col_types <- set_col_types(grouping_variables, "ddddccccddcl") +port_col_types <- set_col_types(grouping_variables, "ddddccccddclc") ################## ##### EQUITY ##### ################## @@ -12,9 +12,9 @@ if (file.exists(equity_input_file)) { port_raw_all_eq <- read_rds(equity_input_file) %>% mutate(id = as.character(id)) - # if (length(colnames(port_raw_all_eq)) != nchar(port_col_types)) { - # stop("Check port_col_types: difference in length") - # } + if (length(colnames(port_raw_all_eq)) != nchar(port_col_types)) { + stop("Check port_col_types: difference in length") + } ald_scen_eq <- get_ald_scen("Equity") @@ -100,9 +100,9 @@ if (file.exists(bonds_inputs_file)) { port_raw_all_cb <- read_rds(bonds_inputs_file) %>% mutate(id = as.character(id)) - # if (length(colnames(port_raw_all_cb)) != nchar(port_col_types)) { - # stop("Check port_col_types: difference in length") - # } + if (length(colnames(port_raw_all_cb)) != nchar(port_col_types)) { + stop("Check port_col_types: difference in length") + } ald_scen_cb <- get_ald_scen("Bonds") From dd87c7171d17e6d75fa5e130dd6d804bc5851060 Mon Sep 17 00:00:00 2001 From: Clare2D Date: Thu, 18 Feb 2021 09:28:01 +0100 Subject: [PATCH 02/12] change list name --- 0_portfolio_input_check_functions.R | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/0_portfolio_input_check_functions.R b/0_portfolio_input_check_functions.R index 0d916916..4c822db8 100644 --- a/0_portfolio_input_check_functions.R +++ b/0_portfolio_input_check_functions.R @@ -5,9 +5,9 @@ read_raw_portfolio_file <- function(project_name) { portfolio <- NA input_path <- paste0(project_location, "/20_Raw_Inputs/") - # pattern matches e.g.: "whatever_Input.csv" - csv_to_read <- list.files(path = input_path, pattern = "_Input[.]csv$") - txt_to_read <- list.files(path = input_path, pattern = "_Input[.]txt$") + + csv_to_read <- list.files(path = input_path, pattern = "_Input.csv") + txt_to_read <- list.files(path = input_path, pattern = "_Input.txt") if (length(csv_to_read) == 1) { @@ -418,7 +418,7 @@ check_fin_mapped_sectors <- function(fin_data) { )) actual_sectors <- unique(fin_data$security_mapped_sector) - if (any(!actual_sectors %in% c(sector_list, other_sector_list, "Other"))) { + if (any(!actual_sectors %in% c(sector_list, pacta_sectors_not_analysed, "Other"))) { stop("Additional Sectors in fin_data") } @@ -1531,7 +1531,7 @@ get_average_emission_data <- function(inc_emission_factors) { average_sector_intensity <- data.frame() if (inc_emission_factors) { - average_sector_intensity <- read_rda(file.path(analysis_inputs_path, "average_sector_intensity.rda")) + average_sector_intensity <- read_rda(paste0(analysis_inputs_path, "/average_sector_intensity.rda")) } return(average_sector_intensity) } @@ -1540,7 +1540,7 @@ get_company_emission_data <- function(inc_emission_factors) { company_emissions <- data.frame() if (inc_emission_factors) { - company_emissions <- read_rda(file.path(analysis_inputs_path, "company_emissions.rda")) + company_emissions <- read_rda(paste0(analysis_inputs_path, "/company_emissions.rda")) } return(company_emissions) } From 985a2b47d595fd2db1e3898f33621d76d1c05dc9 Mon Sep 17 00:00:00 2001 From: Clare2D Date: Thu, 18 Feb 2021 09:28:29 +0100 Subject: [PATCH 03/12] remove unnecessary parameter --- 2_project_input_analysis.R | 3 --- 1 file changed, 3 deletions(-) diff --git a/2_project_input_analysis.R b/2_project_input_analysis.R index 7e342aeb..f45f2895 100644 --- a/2_project_input_analysis.R +++ b/2_project_input_analysis.R @@ -16,9 +16,6 @@ # Obtains data, processes the portfolio and saves the files -# todo: move this to parameter files -other_sector_list <- c("Steel", "Cement", "Shipping", "Aviation") - #################### #### DATA FILES #### #################### From 5954837f58fb3036feafa3a73148bb9d9847bba6 Mon Sep 17 00:00:00 2001 From: Clare2D Date: Thu, 18 Feb 2021 09:34:19 +0100 Subject: [PATCH 04/12] change parameter file name --- 0_global_functions.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/0_global_functions.R b/0_global_functions.R index 08b9b28c..418e238b 100644 --- a/0_global_functions.R +++ b/0_global_functions.R @@ -276,7 +276,7 @@ copy_files <- function(project_name) { input_file <- paste0(raw_input_path, "/", project_name, "_Input.csv") parameter_file <- paste0(par_file_path, "/ReportParameters.yml") - yml_file <- paste0(par_file_path, "/AnalysisParameters.yml") + yml_file <- paste0(par_file_path, "/ProjectParameters.yml") if (!file.exists(input_file)) { file.copy(paste0(folder_location, "/ProjectName_Input.csv"), input_file, overwrite = F) @@ -287,7 +287,7 @@ copy_files <- function(project_name) { } if (!file.exists(yml_file)) { - file.copy(paste0(folder_location, "/AnalysisParameters.yml"), yml_file, overwrite = F) + file.copy(paste0(folder_location, "/ProjectParameters.yml"), yml_file, overwrite = F) } } From ced8110f063394e16e4450c672d570ebf029d84c Mon Sep 17 00:00:00 2001 From: Clare2D Date: Thu, 18 Feb 2021 09:34:42 +0100 Subject: [PATCH 05/12] remove unnecessary text --- 2_project_input_analysis.R | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/2_project_input_analysis.R b/2_project_input_analysis.R index f45f2895..8dafdff4 100644 --- a/2_project_input_analysis.R +++ b/2_project_input_analysis.R @@ -1,20 +1,4 @@ -## Project start -# File List Check for External Data Requirements: -# security_financial_data.rda -# consolidated_financial_data.rda -# debt_financial_data.rda -# bonds_ald_scenario.rda -# equity_ald_scenario.rda -# masterdata_ownership_datastore.rda -# masterdata_debt_datastore.rda - -# To do: write test to see that these files exist - -# optional: -# fund_data_2019Q4.rda (or relevant time stamp) -# revenue_data_member_ticker.rda (if not available, set has_revenue = FALSE in parameter file) - -# Obtains data, processes the portfolio and saves the files +# This script obtains data, processes the portfolio and saves the files #################### #### DATA FILES #### From 2d2a54e6e6c69fc95b1e23a85221cfeb689d04e2 Mon Sep 17 00:00:00 2001 From: Clare2D Date: Thu, 18 Feb 2021 09:42:23 +0100 Subject: [PATCH 06/12] ran styler on repo --- 0_global_functions.R | 7 +- 0_portfolio_input_check_functions.R | 224 +++++++++++++++------------- 0_portfolio_test.R | 13 +- 0_web_functions.R | 57 +++---- R/get_param.R | 1 - R/utils.R | 4 +- test_test_cases.R | 39 +++-- web_tool_script_1.R | 45 +++--- web_tool_script_2.R | 9 +- web_tool_script_3.R | 78 +++++----- 10 files changed, 259 insertions(+), 218 deletions(-) diff --git a/0_global_functions.R b/0_global_functions.R index 418e238b..a8d9224d 100644 --- a/0_global_functions.R +++ b/0_global_functions.R @@ -43,7 +43,7 @@ set_col_types <- function(grouping_variables, fixed_col_types) { return(port_col_types) } -set_project_parameters <- function(file_path){ +set_project_parameters <- function(file_path) { cfg <- config::get(file = file_path) project_report_name <<- cfg$reporting$project_report_name @@ -124,7 +124,6 @@ set_project_parameters <- function(file_path){ inc_stresstest <<- FALSE warning("Warning: inc_stresstest set to standard value (FALSE) as it is not defined in the parameter file") } - } set_global_parameters <- function(file_path) { @@ -251,7 +250,6 @@ set_git_path <- function() { } set_analysis_inputs_path <- function(twodii_internal, data_location_ext, dataprep_ref = datastore_timestamp) { - if (twodii_internal) { analysis_inputs_path <- r2dii.utils::path_dropbox_2dii("PortCheck", "00_Data", "07_AnalysisInputs", dataprep_ref) analysis_inputs_path <- file.path(analysis_inputs_path) @@ -364,6 +362,5 @@ write_log <- function(msg, file_path = log_path, ...) { as.character(msg), ... ) - write(composed, file = file.path(file_path,"error_messages.txt"), append = TRUE) + write(composed, file = file.path(file_path, "error_messages.txt"), append = TRUE) } - diff --git a/0_portfolio_input_check_functions.R b/0_portfolio_input_check_functions.R index 4c822db8..a6aa2f54 100644 --- a/0_portfolio_input_check_functions.R +++ b/0_portfolio_input_check_functions.R @@ -59,44 +59,54 @@ clean_portfolio_col_types <- function(portfolio, grouping_variables) { # portfolio[,grouping_variables] <- lapply(portfolio[,grouping_variables], clean_punctuation) if (is.character(portfolio$investor_name) == FALSE) { - write_log(msg = paste0( - "Wrong variable class for investor_name. Should be character, but is ", - class(portfolio$investor_name), - ". This can introduce errors in further calculations!" - ), - file_path = log_path) + write_log( + msg = paste0( + "Wrong variable class for investor_name. Should be character, but is ", + class(portfolio$investor_name), + ". This can introduce errors in further calculations!" + ), + file_path = log_path + ) } if (is.character(portfolio$portfolio_name) == FALSE) { - write_log(msg = paste0( - "Wrong variable class for portfolio_name Should be character, but is ", - class(portfolio$portfolio_name), - ". This can introduce errors in further calculations!" - ), - file_path = log_path) + write_log( + msg = paste0( + "Wrong variable class for portfolio_name Should be character, but is ", + class(portfolio$portfolio_name), + ". This can introduce errors in further calculations!" + ), + file_path = log_path + ) } if (is.numeric(portfolio$market_value) == FALSE) { - write_log(msg = paste0( - "Wrong variable class for market_value Should be numeric, but is ", - class(portfolio$market_value), - ". This can introduce errors in further calculations!" - ), - file_path = log_path) + write_log( + msg = paste0( + "Wrong variable class for market_value Should be numeric, but is ", + class(portfolio$market_value), + ". This can introduce errors in further calculations!" + ), + file_path = log_path + ) } if (is.character(portfolio$currency) == FALSE) { - write_log(msg = paste0( - "Wrong variable class for currency Should be character, but is ", - class(portfolio$currency), - ". This can introduce errors in further calculations!" - ), - file_path = log_path) + write_log( + msg = paste0( + "Wrong variable class for currency Should be character, but is ", + class(portfolio$currency), + ". This can introduce errors in further calculations!" + ), + file_path = log_path + ) } if (is.character(portfolio$isin) == FALSE) { - write_log(msg = paste0( - "Wrong variable class for isin Should be character, but is ", - class(portfolio$isin), - ". This can introduce errors in further calculations!" - ), - file_path = log_path) + write_log( + msg = paste0( + "Wrong variable class for isin Should be character, but is ", + class(portfolio$isin), + ". This can introduce errors in further calculations!" + ), + file_path = log_path + ) } ### what about number_of_shares??? @@ -112,13 +122,15 @@ clean_portfolio_col_types <- function(portfolio, grouping_variables) { clear_portfolio_input_blanks <- function(portfolio) { if (any(portfolio[, grouping_variables] == "" | is.na(portfolio[, grouping_variables]))) { print("Warning: missing grouping variables, corresponding rows removed") - write_log(msg = paste( - "Warning: some entries of the uploaded portfolio file were removed + write_log( + msg = paste( + "Warning: some entries of the uploaded portfolio file were removed because of missing values in at least one of the variables", str_c(grouping_variables, collapse = ", "), - "\n To ensure complete analysis, please upload a file without + "\n To ensure complete analysis, please upload a file without missing values in these columns." - ), - file_path = log_path) + ), + file_path = log_path + ) portfolio <- portfolio %>% filter_at( grouping_variables, all_vars(!is.na(.)) @@ -230,18 +242,18 @@ map_security_sectors <- function(fin_data, sector_bridge) { fin_data <- fin_data %>% left_join(sector_bridge %>% filter(source == "BICS") %>% select(-source), - by = c("security_bics_subgroup" = "industry_classification") + by = c("security_bics_subgroup" = "industry_classification") ) %>% mutate(security_icb_subsector = as.character(security_icb_subsector)) fin_data_na <- fin_data %>% filter(is.na(sector)) %>% - select(-c(sector,sector_boe,subsector_boe,sector_dnb,sector_ipr,subsector_ipr)) + select(-c(sector, sector_boe, subsector_boe, sector_dnb, sector_ipr, subsector_ipr)) fin_data <- fin_data %>% filter(!is.na(sector)) fin_data_na <- fin_data_na %>% left_join(sector_bridge %>% filter(source == "ICB") %>% select(-source), - by = c("security_icb_subsector" = "industry_classification") + by = c("security_icb_subsector" = "industry_classification") ) fin_data <- fin_data %>% bind_rows(fin_data_na) @@ -267,7 +279,7 @@ map_comp_sectors <- function(comp_fin_data, sector_bridge) { initial_no_rows <- nrow(comp_fin_data) comp_fin_data <- comp_fin_data %>% left_join(sector_bridge %>% filter(source == "BICS") %>% select(-source), - by = c("bics_subgroup" = "industry_classification") + by = c("bics_subgroup" = "industry_classification") ) comp_fin_data_na <- comp_fin_data %>% @@ -277,7 +289,7 @@ map_comp_sectors <- function(comp_fin_data, sector_bridge) { comp_fin_data <- comp_fin_data %>% filter(!is.na(sector)) comp_fin_data_na <- comp_fin_data_na %>% left_join(sector_bridge %>% filter(source == "ICB") %>% select(-source), - by = c("icb_subgroup" = "industry_classification") + by = c("icb_subgroup" = "industry_classification") ) comp_fin_data <- comp_fin_data %>% bind_rows(comp_fin_data_na) @@ -874,7 +886,9 @@ get_and_clean_currency_data <- function() { currencies <- set_currency_timestamp(currencies) - if (all(currencies$exchange_rate) == 1){print("currency data temporary. todo: update")} + if (all(currencies$exchange_rate) == 1) { + print("currency data temporary. todo: update") + } currencies } @@ -924,7 +938,7 @@ get_and_clean_fin_data <- function(fund_data) { } overrides <- read_csv("data/fin_sector_overrides.csv", - col_types = "ccdc" + col_types = "ccdc" ) sector_bridge <- read_csv("data/sector_bridge.csv", col_types = "cccccccc") @@ -1036,7 +1050,7 @@ process_raw_portfolio <- function(portfolio_raw, fund_data, currencies, grouping_variables, - total_fund_list=NA) { + total_fund_list = NA) { portfolio <- clean_colnames_portfolio_input_file(portfolio_raw) portfolio <- clear_portfolio_input_blanks(portfolio) @@ -1072,7 +1086,9 @@ process_raw_portfolio <- function(portfolio_raw, original_value_usd <- sum(portfolio$value_usd, na.rm = T) # correct Funds classification by comparing isin to the list of all known funds isins - if(!is.na(total_fund_list)){portfolio <- portfolio %>% mutate(asset_type = ifelse(is.element(isin, total_fund_list$fund_isin), "Funds", asset_type))} + if (!is.na(total_fund_list)) { + portfolio <- portfolio %>% mutate(asset_type = ifelse(is.element(isin, total_fund_list$fund_isin), "Funds", asset_type)) + } # identify funds in the portfolio fund_portfolio <- identify_fund_portfolio(portfolio) @@ -1167,19 +1183,19 @@ get_fund_coverage <- function(portfolio_raw, fund_portfolio_total <- portfolio_total %>% filter(!is.na(fund_isin)) - fund_portfolio_total_mapped_value_usd <- fund_portfolio_total %>% + fund_portfolio_total_mapped_value_usd <- fund_portfolio_total %>% group_by(holding_id) %>% summarize(total_mapped_value_usd = sum(value_usd)) - fund_portfolio_missing_value_usd <- fund_portfolio_total %>% - filter(nchar(isin)!=12) %>% + fund_portfolio_missing_value_usd <- fund_portfolio_total %>% + filter(nchar(isin) != 12) %>% group_by(holding_id) %>% summarize(missing_value_usd = sum(value_usd)) - fund_portfolio_funds_in_funds_not_mapped_value_usd <- fund_portfolio_total %>% - filter(nchar(isin)==12 & asset_type == "Funds") %>% + fund_portfolio_funds_in_funds_not_mapped_value_usd <- fund_portfolio_total %>% + filter(nchar(isin) == 12 & asset_type == "Funds") %>% group_by(holding_id) %>% summarize(funds_in_funds_not_mapped = sum(value_usd)) @@ -1201,7 +1217,7 @@ get_fund_coverage <- function(portfolio_raw, fund_portfolio <- fund_portfolio %>% mutate(effective_coverage = (total_mapped_value_usd - missing_value_usd - funds_in_funds_not_mapped) / value_usd) fund_portfolio <- fund_portfolio %>% mutate(fund_data_file_coverage = (total_mapped_value_usd) / value_usd) fund_portfolio <- fund_portfolio %>% mutate(lipper_data_coverage = (total_mapped_value_usd - missing_value_usd) / total_mapped_value_usd) - fund_portfolio <- fund_portfolio %>% mutate(lost_coverage_fif = 1 - (total_mapped_value_usd - missing_value_usd - funds_in_funds_not_mapped) / (total_mapped_value_usd-missing_value_usd)) + fund_portfolio <- fund_portfolio %>% mutate(lost_coverage_fif = 1 - (total_mapped_value_usd - missing_value_usd - funds_in_funds_not_mapped) / (total_mapped_value_usd - missing_value_usd)) @@ -1211,30 +1227,33 @@ get_fund_coverage <- function(portfolio_raw, } -summarize_fund_coverage <- function(fund_portfolio) { - - fund_portfolio %>% select(investor_name, - portfolio_name, - isin, - value_usd, - company_name, - effective_coverage) +summarize_fund_coverage <- function(fund_portfolio) { + fund_portfolio %>% select( + investor_name, + portfolio_name, + isin, + value_usd, + company_name, + effective_coverage + ) return(fund_portfolio) } -list_unknown_funds_in_funds <- function(portfolio_total) { - +list_unknown_funds_in_funds <- function(portfolio_total) { fund_portfolio_total <- portfolio_total %>% filter(!is.na(fund_isin)) table_of_funds_in_funds_not_mapped <- fund_portfolio_total %>% - filter(nchar(isin)==12 & asset_type == "Funds") %>% - mutate(direct_holding="FALSE") %>% - select(investor_name, - portfolio_name, - isin, - value_usd, - company_name, - fund_isin) %>% rename(parent_fund_isin = fund_isin) + filter(nchar(isin) == 12 & asset_type == "Funds") %>% + mutate(direct_holding = "FALSE") %>% + select( + investor_name, + portfolio_name, + isin, + value_usd, + company_name, + fund_isin + ) %>% + rename(parent_fund_isin = fund_isin) return(table_of_funds_in_funds_not_mapped) } @@ -1290,7 +1309,6 @@ create_merged_portfolio <- function(eq_portfolio, cb_portfolio) { } create_portfolio_subset <- function(portfolio, portfolio_type) { - if (portfolio_type %in% unique(portfolio$asset_type)) { portfolio_subset <- portfolio %>% ungroup() %>% @@ -1341,7 +1359,7 @@ create_audit_chart <- function(audit_file, proc_input_path) { flag_categories <- names(flag_numbers_colors) flag_numbers$flag <- factor(flag_numbers$flag, - levels = flag_categories + levels = flag_categories ) y_intercept <- sum(flag_numbers$perc_n[flag_numbers$flag != "Included in analysis"]) @@ -1349,13 +1367,13 @@ create_audit_chart <- function(audit_file, proc_input_path) { nrows <- length(unique(flag_numbers$flag)) nrow_Legend <- ifelse(nrows <= 2, 3.2, - ifelse(nrows == 3 | nrows == 4, 3.7, - ifelse(nrows == 5, 3.8, - ifelse(nrows == 6, 4.0, - ifelse(nrows == 7, 4, 4.6) - ) - ) - ) + ifelse(nrows == 3 | nrows == 4, 3.7, + ifelse(nrows == 5, 3.8, + ifelse(nrows == 6, 4.0, + ifelse(nrows == 7, 4, 4.6) + ) + ) + ) ) ## Chart @@ -1390,10 +1408,10 @@ create_audit_chart <- function(audit_file, proc_input_path) { guides(fill = guide_legend(reverse = TRUE, ncol = 1)) + geom_hline(aes(yintercept = y_intercept), color = "#3D3D3C", size = 1, show.legend = FALSE) + geom_text(aes(label = "Not included\nin analysis\n\n", x = 1.5, y_intercept - 0.02), - color = "#2e2e2e", family = base_family, size = base_size / 2.5, hjust = 1 + color = "#2e2e2e", family = base_family, size = base_size / 2.5, hjust = 1 ) + geom_text(aes(label = "Included\nin analysis\n\n", x = 1.5, y_intercept + 0.02), - color = "#207ddb", family = base_family, size = base_size / 2.5, hjust = 0 + color = "#207ddb", family = base_family, size = base_size / 2.5, hjust = 0 ) @@ -1443,7 +1461,6 @@ clean_unmatched_holdings <- function(portfolio) { calculate_average_portfolio_emissions <- function(portfolio_total, comp_fin_data, average_sector_intensity) { - min_portfolio <- portfolio_total %>% select( investor_name, @@ -1466,7 +1483,7 @@ calculate_average_portfolio_emissions <- function(portfolio_total, financial_sector, bics_sector ) %>% - summarise(value_usd = sum(value_usd, na.rm = T), .groups = "drop") %>% + summarise(value_usd = sum(value_usd, na.rm = T), .groups = "drop") %>% left_join( select( average_sector_intensity, @@ -1498,7 +1515,7 @@ calculate_average_portfolio_emissions <- function(portfolio_total, mean_intensity.x ) ) %>% - select(-mean_intensity.x, -mean_intensity.y) %>% + select(-mean_intensity.x, -mean_intensity.y) %>% mutate(weighted_sector_emissions = value_usd * mean_intensity) min_portfolio <- min_portfolio %>% @@ -1514,7 +1531,7 @@ calculate_average_portfolio_emissions <- function(portfolio_total, sector ) ) %>% - group_by(investor_name, portfolio_name, asset_type,sector) %>% + group_by(investor_name, portfolio_name, asset_type, sector) %>% summarise( weighted_sector_emissions = sum(weighted_sector_emissions, na.rm = T), .groups = "drop" @@ -1546,11 +1563,11 @@ get_company_emission_data <- function(inc_emission_factors) { } prepare_portfolio_emissions <- function( - audit_file, - fin_data, - comp_fin_data, - average_sector_intensity, - company_emissions) { + audit_file, + fin_data, + comp_fin_data, + average_sector_intensity, + company_emissions) { audit_file <- audit_file %>% janitor::clean_names(case = "snake") @@ -1714,12 +1731,12 @@ prepare_portfolio_emissions <- function( } calculate_portfolio_emissions <- function( - inc_emission_factors, - audit_file, - fin_data, - comp_fin_data, - average_sector_intensity, - company_emissions) { + inc_emission_factors, + audit_file, + fin_data, + comp_fin_data, + average_sector_intensity, + company_emissions) { audit_sector_emissions <- data.frame() if (inc_emission_factors) { @@ -1781,24 +1798,19 @@ add_other_to_sector_classifications <- function(audit) { } -pw_calculations <- function(eq_portfolio, cb_portfolio){ - +pw_calculations <- function(eq_portfolio, cb_portfolio) { port_all <- data.frame() - if(data_check(eq_portfolio)){ - + if (data_check(eq_portfolio)) { port_all <- bind_rows(port_all, eq_portfolio) } - if(data_check(cb_portfolio)){ - + if (data_check(cb_portfolio)) { port_all <- bind_rows(port_all, cb_portfolio) - } - if(data_check(port_all)){ - - port_all <- port_all %>% select(!!!rlang::syms(grouping_variables),company_id, value_usd) + if (data_check(port_all)) { + port_all <- port_all %>% select(!!!rlang::syms(grouping_variables), company_id, value_usd) port_all <- calculate_port_weight(port_all, grouping_variables) @@ -1808,11 +1820,9 @@ pw_calculations <- function(eq_portfolio, cb_portfolio){ summarise(port_weight = sum(port_weight, na.rm = T), .groups = "drop") %>% select(company_id, port_weight) %>% rename(portfolio_weight = port_weight) - - }else{ + } else { pw <- data.frame(company_id = "No companies in portfolio", portfolio_weight = "0") } return(pw) - } diff --git a/0_portfolio_test.R b/0_portfolio_test.R index 34db0071..89eaad8f 100644 --- a/0_portfolio_test.R +++ b/0_portfolio_test.R @@ -81,7 +81,6 @@ calculate_ownership_weight <- function(portfolio) { } calculate_port_weight <- function(portfolio, grouping_variables) { - portfolio <- portfolio %>% ungroup() %>% group_by(!!!rlang::syms(grouping_variables)) %>% @@ -167,8 +166,10 @@ aggregate_portfolio <- function(df) { plan_tech_prod, plan_alloc_wt_tech_prod, plan_carsten, plan_emission_factor, scen_tech_prod, scen_alloc_wt_tech_prod, scen_carsten, scen_emission_factor ) %>% - mutate(plan_emission_factor = ifelse(is.na(plan_emission_factor), 0, plan_emission_factor), - scen_emission_factor = ifelse(is.na(scen_emission_factor), 0, scen_emission_factor)) %>% + mutate( + plan_emission_factor = ifelse(is.na(plan_emission_factor), 0, plan_emission_factor), + scen_emission_factor = ifelse(is.na(scen_emission_factor), 0, scen_emission_factor) + ) %>% group_by( !!!rlang::syms(grouping_variables), scenario, allocation, equity_market, scenario_geography, year, @@ -211,8 +212,10 @@ aggregate_map_data <- function(portfolio) { ald_sector, technology, financial_sector, allocation, allocation_weight, ald_production_unit ) %>% - summarise(plan_alloc_wt_tech_prod = sum(plan_alloc_wt_tech_prod, na.rm = TRUE), - .groups = "drop_last") %>% + summarise( + plan_alloc_wt_tech_prod = sum(plan_alloc_wt_tech_prod, na.rm = TRUE), + .groups = "drop_last" + ) %>% mutate(plan_alloc_wt_sec_prod = sum(plan_alloc_wt_tech_prod)) if (data_check(portfolio)) { diff --git a/0_web_functions.R b/0_web_functions.R index 1e830d76..7b9ef50f 100644 --- a/0_web_functions.R +++ b/0_web_functions.R @@ -106,7 +106,6 @@ add_naming_to_portfolio <- function(portfolio_raw) { } get_input_files <- function(portfolio_name_ref_all) { - portfolio <- tibble() input_path <- file.path(project_location, "20_Raw_Inputs") @@ -221,14 +220,14 @@ read_web_input_file <- function(input_file_path) { if (data_check(input_file) == FALSE) { warning("Input file not readable") ifelse(nrow(input_file) == 0, - write_log( - msg = "Input file has 0 rows. Please ensure the uploaded file is not empty.", - file_path = log_path - ), - write_log( - msg = "Input file could not be transformed into a data.frame. Please check the uploaded file has the correct format.", - file_path = log_path - ) + write_log( + msg = "Input file has 0 rows. Please ensure the uploaded file is not empty.", + file_path = log_path + ), + write_log( + msg = "Input file could not be transformed into a data.frame. Please check the uploaded file has the correct format.", + file_path = log_path + ) ) } @@ -306,7 +305,7 @@ save_cleaned_files <- function(save_loc, debt_fin_data, average_sector_intensity, company_emissions, - total_fund_list=NA) { + total_fund_list = NA) { if (!dir.exists(save_loc)) { dir.create(save_loc) } @@ -318,8 +317,9 @@ save_cleaned_files <- function(save_loc, fst::write_fst(debt_fin_data, file.path(save_loc, "debt_fin_data.fst")) fst::write_fst(average_sector_intensity, file.path(save_loc, "average_sector_intensity.fst")) fst::write_fst(company_emissions, file.path(save_loc, "company_emissions.fst")) - if (!is.na(total_fund_list)) - {fst::write_fst(total_fund_list, file.path(save_loc, "total_fund_list.fst"))} + if (!is.na(total_fund_list)) { + fst::write_fst(total_fund_list, file.path(save_loc, "total_fund_list.fst")) + } if (check_file_size(save_loc)) warning("File size exceeds what can be pushed to GitHub. Check before Committing") } @@ -329,7 +329,7 @@ check_file_size <- function(folder_to_check) { any(file.size(files_to_check) > 100e6) } -empty_portfolio_results <- function(){ +empty_portfolio_results <- function() { tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "scenario" = NA_character_, "allocation" = NA_character_, @@ -348,7 +348,7 @@ empty_portfolio_results <- function(){ ) } -empty_company_results <- function(){ +empty_company_results <- function() { tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "scenario" = NA_character_, "allocation" = NA_character_, @@ -371,19 +371,23 @@ empty_company_results <- function(){ ) } -empty_emissions_results <- function(){ - tibble("investor_name" = NA_character_, "portfolio_name" = NA_character_, - "asset_type" = NA_character_, "sector" = NA_character_, - "weighted_sector_emissions" = NA_real_) +empty_emissions_results <- function() { + tibble( + "investor_name" = NA_character_, "portfolio_name" = NA_character_, + "asset_type" = NA_character_, "sector" = NA_character_, + "weighted_sector_emissions" = NA_real_ + ) } -empty_audit_file <- function(){ - tibble("investor_name" = NA_character_, "portfolio_name" = NA_character_, - "asset_type" = NA_character_, "valid_input" = NA, "isin" = NA_character_, - "direct_holding" = NA, "value_usd" = NA_real_) +empty_audit_file <- function() { + tibble( + "investor_name" = NA_character_, "portfolio_name" = NA_character_, + "asset_type" = NA_character_, "valid_input" = NA, "isin" = NA_character_, + "direct_holding" = NA, "value_usd" = NA_real_ + ) } -empty_map_results <- function(){ +empty_map_results <- function() { tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "ald_location" = NA_character_, "year" = NA_integer_, @@ -396,7 +400,7 @@ empty_map_results <- function(){ ) } -empty_st_results <- function(){ +empty_st_results <- function() { tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "ald_sector" = NA_character_, "technology" = NA_character_, @@ -410,7 +414,7 @@ empty_st_results <- function(){ ) } -empty_ipr_st_results <- function(){ +empty_ipr_st_results <- function() { tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "sector" = NA_character_, "subsector" = NA_character_, @@ -420,7 +424,7 @@ empty_ipr_st_results <- function(){ ) } -empty_portfolio_overview <- function(){ +empty_portfolio_overview <- function() { tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "asset_type" = NA_character_, "financial_sector" = NA_character_, @@ -428,4 +432,3 @@ empty_portfolio_overview <- function(){ "asset_value_usd" = NA_real_, "portfolio_value_usd" = NA_real_ ) } - diff --git a/R/get_param.R b/R/get_param.R index 65bd5b67..f6bf3831 100644 --- a/R/get_param.R +++ b/R/get_param.R @@ -36,4 +36,3 @@ pluck_param <- function(x, file, if_null) { out } - diff --git a/R/utils.R b/R/utils.R index 8432f5f3..2caa813f 100644 --- a/R/utils.R +++ b/R/utils.R @@ -76,7 +76,7 @@ dockerfile_packages <- function(path = packages_path()) { c( ' && Rscript -e "install.packages( \\', - paste0(' ', format_as_vector(pkg), ' \\'), + paste0(" ", format_as_vector(pkg), " \\"), ' )" \\' ) } @@ -84,7 +84,7 @@ dockerfile_packages <- function(path = packages_path()) { format_as_vector <- function(string) { x <- glue("'{string}',") x[length(x)] <- sub(",$", "", x[length(x)]) - c('c(', glue(" {x}"), ')' ) + c("c(", glue(" {x}"), ")") } packages_path <- function() { diff --git a/test_test_cases.R b/test_test_cases.R index 014f2f82..b21e0cd6 100644 --- a/test_test_cases.R +++ b/test_test_cases.R @@ -20,8 +20,12 @@ for (i in seq_along(test_cases_csvs)) { investor_name <- unique(test_case$Investor.Name) # if no unique, valid portfolio_name or investor_name, use the filename - if (length(portfolio_name) != 1) { portfolio_name <- filename } - if (length(investor_name) != 1) { investor_name <- filename } + if (length(portfolio_name) != 1) { + portfolio_name <- filename + } + if (length(investor_name) != 1) { + investor_name <- filename + } yaml_data <- list(default = list( @@ -41,8 +45,10 @@ for (i in seq_along(test_cases_csvs)) { fs::dir_create(out_dir, recurse = TRUE) sub_directories_needed <- - c("00_Log_Files", "10_Parameter_File", "20_Raw_Inputs", - "30_Processed_Inputs", "40_Results", "50_Outputs") + c( + "00_Log_Files", "10_Parameter_File", "20_Raw_Inputs", + "30_Processed_Inputs", "40_Results", "50_Outputs" + ) lapply(sub_directories_needed, function(sub_dir) { fs::dir_create(fs::path(out_dir, sub_dir), recurse = TRUE) @@ -50,14 +56,16 @@ for (i in seq_along(test_cases_csvs)) { write_csv(test_case, fs::path(out_dir, "20_Raw_Inputs", paste0(portfolio_name, ".csv"))) write_yaml(yaml_data, fs::path(out_dir, "10_Parameter_File", paste0(portfolio_name, "_PortfolioParameters.yml")), indent = 4) - fs::file_copy("working_dir/10_Parameter_File/AnalysisParameters.yml", - fs::path(out_dir, "10_Parameter_File", "AnalysisParameters.yml")) + fs::file_copy( + "working_dir/10_Parameter_File/AnalysisParameters.yml", + fs::path(out_dir, "10_Parameter_File", "AnalysisParameters.yml") + ) } report_git_status <- function(repo_roots = ".") { - for (repo_root in repo_roots) { + for (repo_root in repo_roots) { cli::cli_h1(paste0("repo status for: ", repo_root)) system2("git", c("-C", repo_root, "fetch"), stdout = FALSE) @@ -83,8 +91,10 @@ report_git_status <- owner <- "2DegreesInvesting" repo_name <- basename(repo_root) open_pr_list <- - gh::gh("/repos/:owner/:repo/pulls", owner = owner, repo = repo_name, - state = "open", .limit = Inf) + gh::gh("/repos/:owner/:repo/pulls", + owner = owner, repo = repo_name, + state = "open", .limit = Inf + ) if (length(open_pr_list) > 0) { infos <- @@ -97,7 +107,8 @@ report_git_status <- } report_git_status( - c("../PACTA_analysis", + c( + "../PACTA_analysis", "../create_interactive_report", "../StressTestingModelDev", "../pacta-data", @@ -114,10 +125,14 @@ for (csv_num in seq_along(test_cases_csvs)) { sub_directory <- portfolio_name # if no unique, valid portfolio_name, use an empty string - if (length(portfolio_name) != 1) { portfolio_name <- "" } + if (length(portfolio_name) != 1) { + portfolio_name <- "" + } # if no unique, valid portfolio_name, use the filename - if (length(sub_directory) != 1) { sub_directory <- filename } + if (length(sub_directory) != 1) { + sub_directory <- filename + } out_dir <- fs::path(test_cases_output_dir, sub_directory) diff --git a/web_tool_script_1.R b/web_tool_script_1.R index e606ba28..dd065f79 100644 --- a/web_tool_script_1.R +++ b/web_tool_script_1.R @@ -9,8 +9,12 @@ source("0_web_functions.R") source("0_json_functions.R") source("0_portfolio_test.R") -if (!exists("portfolio_name_ref_all")) { portfolio_name_ref_all <- "TestPortfolio_Input" } -if (!exists("portfolio_root_dir")) { portfolio_root_dir <- "working_dir" } +if (!exists("portfolio_name_ref_all")) { + portfolio_name_ref_all <- "TestPortfolio_Input" +} +if (!exists("portfolio_root_dir")) { + portfolio_root_dir <- "working_dir" +} portfolio_root_dir <- "working_dir" setup_project() @@ -21,7 +25,7 @@ set_webtool_paths(portfolio_root_dir) set_portfolio_parameters(file_path = fs::path(par_file_path, paste0(portfolio_name_ref_all, "_PortfolioParameters.yml"))) -set_project_parameters(file.path(working_location, "parameter_files",paste0("ProjectParameters_", project_code, ".yml"))) +set_project_parameters(file.path(working_location, "parameter_files", paste0("ProjectParameters_", project_code, ".yml"))) # need to define an alternative location for data files analysis_inputs_path <- set_analysis_inputs_path(twodii_internal, data_location_ext, dataprep_timestamp) @@ -43,11 +47,11 @@ file_location <- file.path(analysis_inputs_path, "cleaned_files") if (new_data == TRUE) { currencies <- get_and_clean_currency_data() - - + + total_fund_list <- get_and_clean_total_fund_list_data() - - # fund_data <- get_and_clean_fund_data() + + # fund_data <- get_and_clean_fund_data() fund_data <- data.frame() fin_data <- get_and_clean_fin_data(fund_data) @@ -71,11 +75,11 @@ if (new_data == TRUE) { debt_fin_data, average_sector_intensity, company_emissions, - total_fund_list=total_fund_list + total_fund_list = total_fund_list ) } else { currencies <- fst::read_fst(file.path(file_location, "currencies.fst")) - + read_fst_or_return_null <- function(fst_file) { if (!file.exists(fst_file)) { return(NULL) @@ -86,19 +90,19 @@ if (new_data == TRUE) { fund_data_path <- file.path(file_location, "fund_data.fst") - + fund_data <- read_fst_or_return_null(fund_data_path) - + fund_data$holding_isin <- as.character(fund_data$holding_isin) fund_data$fund_isin <- as.character(fund_data$fund_isin) - - + + fin_data <- fst::read_fst(file.path(file_location, "fin_data.fst")) comp_fin_data <- fst::read_fst(file.path(file_location, "comp_fin_data.fst")) debt_fin_data <- fst::read_fst(file.path(file_location, "debt_fin_data.fst")) - + total_fund_list <- fst::read_fst(file.path(file_location, "total_fund_list.fst")) if (inc_emission_factors) { @@ -125,7 +129,7 @@ portfolio <- process_raw_portfolio( fund_data, currencies, grouping_variables, - total_fund_list=total_fund_list + total_fund_list = total_fund_list ) # information of coverage and coverage loses for all funds in raw_portfolio @@ -168,7 +172,8 @@ audit_file <- create_audit_file(portfolio_total) emissions_totals <- calculate_average_portfolio_emissions( portfolio_total, comp_fin_data, - average_sector_intensity) + average_sector_intensity +) port_weights <- pw_calculations(eq_portfolio, cb_portfolio) @@ -194,10 +199,10 @@ save_if_exists(emissions_totals, portfolio_name, file.path(proc_input_path_, "em save_if_exists(fund_coverage_summary, portfolio_name, file.path(proc_input_path_, "fund_coverage_summary.rda")) save_if_exists(unknown_funds_in_funds, portfolio_name, file.path(proc_input_path_, "unknown_funds_in_funds.rda")) -if(data_check(port_weights)){ - port_weights <- jsonlite::toJSON(x=port_weights) - write(x = port_weights, file = file.path(proc_input_path_,"portfolio_weights.json")) - } +if (data_check(port_weights)) { + port_weights <- jsonlite::toJSON(x = port_weights) + write(x = port_weights, file = file.path(proc_input_path_, "portfolio_weights.json")) +} rm(portfolio_total) rm(portfolio) diff --git a/web_tool_script_2.R b/web_tool_script_2.R index 5710d74b..b9f4e076 100644 --- a/web_tool_script_2.R +++ b/web_tool_script_2.R @@ -11,8 +11,12 @@ source("0_portfolio_test.R") source("0_global_functions.R") source("0_web_functions.R") -if (!exists("portfolio_name_ref_all")) { portfolio_name_ref_all <- "TestPortfolio_Input" } -if (!exists("portfolio_root_dir")) { portfolio_root_dir <- "working_dir" } +if (!exists("portfolio_name_ref_all")) { + portfolio_name_ref_all <- "TestPortfolio_Input" +} +if (!exists("portfolio_root_dir")) { + portfolio_root_dir <- "working_dir" +} setup_project() @@ -130,7 +134,6 @@ bonds_inputs_file <- file.path(proc_input_path, portfolio_name_ref_all, "bonds_p # portfolio_name <- file_names$portfolio_name if (file.exists(bonds_inputs_file)) { - ald_scen_cb <- get_ald_scen("Bonds") ald_raw_cb <- get_ald_raw("Bonds") diff --git a/web_tool_script_3.R b/web_tool_script_3.R index 32a3cafb..8d11291d 100644 --- a/web_tool_script_3.R +++ b/web_tool_script_3.R @@ -6,8 +6,12 @@ use_r_packages() source("0_global_functions.R") source("0_web_functions.R") -if (!exists("portfolio_name_ref_all")) { portfolio_name_ref_all <- "TestPortfolio_Input" } -if (!exists("portfolio_root_dir")) { portfolio_root_dir <- "working_dir" } +if (!exists("portfolio_name_ref_all")) { + portfolio_name_ref_all <- "TestPortfolio_Input" +} +if (!exists("portfolio_root_dir")) { + portfolio_root_dir <- "working_dir" +} setup_project() @@ -15,15 +19,14 @@ set_webtool_paths(portfolio_root_dir) set_portfolio_parameters(file_path = fs::path(par_file_path, paste0(portfolio_name_ref_all, "_PortfolioParameters.yml"))) -set_project_parameters(file.path(working_location, "parameter_files",paste0("ProjectParameters_", project_code, ".yml"))) +set_project_parameters(file.path(working_location, "parameter_files", paste0("ProjectParameters_", project_code, ".yml"))) -if(project_code == "PA2020FL"){ - peer_group = case_when( - peer_group %in% c("other")~ "Others", +if (project_code == "PA2020FL") { + peer_group <- case_when( + peer_group %in% c("other") ~ "Others", peer_group %in% c("bank", "assetmanager") ~ "Banks and Asset Managers", peer_group %in% c("pensionfund", "insurance") ~ "Pension Funds and Insurances" ) - } @@ -33,25 +36,28 @@ source(file.path(template_path, "create_interactive_report.R")) source(file.path(template_path, "create_executive_summary.R")) source(file.path(template_path, "useful_functions.R")) -report_name = select_report_template(project_report_name = project_report_name, - language_select = language_select) +report_name <- select_report_template( + project_report_name = project_report_name, + language_select = language_select +) -exec_summary_name = select_exec_summary_template(project_report_name = project_report_name, - language_select = language_select) +exec_summary_name <- select_exec_summary_template( + project_report_name = project_report_name, + language_select = language_select +) -template_dir <- paste0(template_path, report_name,"/_book/") -exec_summary_dir <- paste0(template_path, exec_summary_name,"/") +template_dir <- paste0(template_path, report_name, "/_book/") +exec_summary_dir <- paste0(template_path, exec_summary_name, "/") survey_dir <- path(user_results_path, project_code, "survey") real_estate_dir <- path(user_results_path, project_code, "real_estate") output_dir <- file.path(outputs_path, portfolio_name_ref_all) -if (file.exists(file.path(proc_input_path, portfolio_name_ref_all, "audit_file.rda"))){ +if (file.exists(file.path(proc_input_path, portfolio_name_ref_all, "audit_file.rda"))) { audit_file <- readRDS(file.path(proc_input_path, portfolio_name_ref_all, "audit_file.rda")) -}else{ +} else { audit_file <- empty_audit_file() - } # load portfolio overview @@ -62,10 +68,11 @@ if (file.exists(file.path(proc_input_path, portfolio_name_ref_all, "overview_por } -if (file.exists(file.path(proc_input_path, portfolio_name_ref_all, "emissions.rda"))){ +if (file.exists(file.path(proc_input_path, portfolio_name_ref_all, "emissions.rda"))) { emissions <- read_rds(file.path(proc_input_path, portfolio_name_ref_all, "emissions.rda")) -}else{ - emissions <- empty_emissions_results()} +} else { + emissions <- empty_emissions_results() +} # load equity portfolio data if (file.exists(file.path(results_path, portfolio_name_ref_all, "Equity_results_portfolio.rda"))) { @@ -131,27 +138,27 @@ if (file.exists(file.path(results_path, portfolio_name_ref_all, "Stress_test_res } # load peers results both individual and aggregate -if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio.rda")))){ +if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio.rda")))) { peers_equity_results_portfolio <- read_rds(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio.rda"))) -}else{ +} else { peers_equity_results_portfolio <- empty_portfolio_results() } -if(file.exists(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio.rda")))){ +if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio.rda")))) { peers_bonds_results_portfolio <- read_rds(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio.rda"))) -}else{ +} else { peers_bonds_results_portfolio <- empty_portfolio_results() } -if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio_ind.rda")))){ +if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio_ind.rda")))) { peers_equity_results_user <- read_rds(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio_ind.rda"))) -}else{ +} else { peers_equity_results_user <- empty_portfolio_results() } -if(file.exists(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio_ind.rda")))){ +if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio_ind.rda")))) { peers_bonds_results_user <- read_rds(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio_ind.rda"))) -}else{ +} else { peers_bonds_results_user <- empty_portfolio_results() } @@ -174,18 +181,18 @@ js_translations <- jsonlite::fromJSON( ) sector_order <- readr::read_csv( - path(template_path, "data","sector_order","sector_order.csv"), + path(template_path, "data", "sector_order", "sector_order.csv"), col_types = cols() ) # Needed for testing only shock <- shock_year # this should come directly from the stress test.. 2030 based on current discussions in CHPA2020 case -select_scenario_auto = scenario_auto -select_scenario_other = scenario_other -select_scenario_shipping = scenario_shipping -twodi_sectors = sector_list -repo_path = template_path -file_name = "template.Rmd" +select_scenario_auto <- scenario_auto +select_scenario_other <- scenario_other +select_scenario_shipping <- scenario_shipping +twodi_sectors <- sector_list +repo_path <- template_path +file_name <- "template.Rmd" create_interactive_report( repo_path = template_path, @@ -234,7 +241,7 @@ create_interactive_report( display_currency = display_currency, currency_exchange_value = currency_exchange_value, header_dictionary = header_dictionary, - sector_order = sector_order + sector_order = sector_order ) @@ -261,5 +268,4 @@ create_executive_summary( peers_bonds_results_portfolio = peers_bonds_results_portfolio, peers_equity_results_user = peers_equity_results_user, peers_bonds_results_user = peers_bonds_results_user - ) From 495f06286a66794238985893db5e7c9e97283661 Mon Sep 17 00:00:00 2001 From: Clare2D Date: Thu, 18 Feb 2021 09:47:04 +0100 Subject: [PATCH 07/12] replace paste0 with file.path --- 0_portfolio_input_check_functions.R | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/0_portfolio_input_check_functions.R b/0_portfolio_input_check_functions.R index a6aa2f54..e7b7b88e 100644 --- a/0_portfolio_input_check_functions.R +++ b/0_portfolio_input_check_functions.R @@ -4,29 +4,29 @@ read_raw_portfolio_file <- function(project_name) { portfolio <- NA - input_path <- paste0(project_location, "/20_Raw_Inputs/") + input_path <- file.path(project_location, "20_Raw_Inputs") csv_to_read <- list.files(path = input_path, pattern = "_Input.csv") txt_to_read <- list.files(path = input_path, pattern = "_Input.txt") if (length(csv_to_read) == 1) { - portfolio <- read_csv(paste0(input_path, csv_to_read)) + portfolio <- read_csv(file.path(input_path, csv_to_read)) } if (length(txt_to_read) == 1) { - enc <- guess_encoding(paste0(input_path, txt_to_read))$encoding[1] - portfolio <- read.table(paste0(input_path, txt_to_read), sep = ",", header = T, fileEncoding = enc) + enc <- guess_encoding(file.path(input_path, txt_to_read))$encoding[1] + portfolio <- read.table(file.path(input_path, txt_to_read), sep = ",", header = T, fileEncoding = enc) } # Reads in Files saved with a ; not a , if (ncol(portfolio) == 1 & length(csv_to_read) == 1) { - portfolio <- read.csv(paste0(input_path, csv_to_read), strip.white = T, stringsAsFactors = F, sep = ";") + portfolio <- read.csv(file.path(input_path, csv_to_read), strip.white = T, stringsAsFactors = F, sep = ";") } if (ncol(portfolio) == 1 & length(txt_to_read) == 1) { - portfolio <- read.table(paste0(input_path, txt_to_read), sep = "\t", header = T, fileEncoding = enc) + portfolio <- read.table(file.path(input_path, txt_to_read), sep = "\t", header = T, fileEncoding = enc) } if (ncol(portfolio) == 1 & length(txt_to_read) == 1) { - portfolio <- read.table(paste0(input_path, txt_to_read), sep = ";", header = T, fileEncoding = enc) + portfolio <- read.table(file.path(input_path, txt_to_read), sep = ";", header = T, fileEncoding = enc) } @@ -896,13 +896,13 @@ get_and_clean_currency_data <- function() { get_and_clean_fund_data <- function() { fund_data <- NA # Fund Data - if (file.exists(paste0(analysis_inputs_path, "/fund_data_", financial_timestamp, ".rda"))) { - fund_data <- readRDS(paste0(analysis_inputs_path, "/fund_data_", financial_timestamp, ".rda")) - } else if (file.exists(paste0(analysis_inputs_path, "/fund_data_2018Q4.rda"))) { - fund_data <- readRDS(paste0(analysis_inputs_path, "/fund_data_2018Q4.rda")) + if (file.exists(file.path(analysis_inputs_path, "fund_data_", financial_timestamp, ".rda"))) { + fund_data <- readRDS(file.path(analysis_inputs_path, "fund_data_", financial_timestamp, ".rda")) + } else if (file.exists(file.path(analysis_inputs_path, "fund_data_2018Q4.rda"))) { + fund_data <- readRDS(file.path(analysis_inputs_path, "fund_data_2018Q4.rda")) print("Old Fund Data being used. Replace FundsData2018Q4 or check name of file.") - } else if (file.exists(paste0(analysis_inputs_path, "/SFC_26052020_funds.csv"))) { - fund_data <- read_csv(paste0(analysis_inputs_path, "/SFC_26052020_funds.csv")) + } else if (file.exists(file.path(analysis_inputs_path, "SFC_26052020_funds.csv"))) { + fund_data <- read_csv(file.path(analysis_inputs_path, "SFC_26052020_funds.csv")) print("2020Q2 SFC fund data being used") } else { if (!data_check(fund_data)) { @@ -923,7 +923,7 @@ get_and_clean_fund_data <- function() { get_and_clean_fin_data <- function(fund_data) { # Financial Data - fin_data_raw <- read_rda(paste0(analysis_inputs_path, "/security_financial_data.rda")) %>% as_tibble() + fin_data_raw <- read_rda(file.path(analysis_inputs_path, "security_financial_data.rda")) %>% as_tibble() # remove unclear duplicates from raw financial data. This should be moved to DataStore. rm_duplicates <- read_csv("non_distinct_isins.csv") @@ -1011,7 +1011,7 @@ get_and_clean_revenue_data <- function() { revenue_data <- data.frame() if (has_revenue) { - revenue_data <- read_rda(paste0(analysis_inputs_path, "/revenue_data_member_ticker.rda")) + revenue_data <- read_rda(file.path(analysis_inputs_path, "revenue_data_member_ticker.rda")) # col_types = "dcdcclcd") revenue_data <- revenue_data %>% @@ -1024,7 +1024,7 @@ get_and_clean_revenue_data <- function() { } get_and_clean_company_fin_data <- function() { - comp_fin_data_raw <- read_rds(paste0(analysis_inputs_path, "/consolidated_financial_data.rda")) + comp_fin_data_raw <- read_rds(file.path(analysis_inputs_path, "consolidated_financial_data.rda")) comp_fin_data_raw <- comp_fin_data_raw %>% select( company_id, company_name, bloomberg_id, country_of_domicile, corporate_bond_ticker, bics_subgroup, bics_sector, @@ -1040,7 +1040,7 @@ get_and_clean_company_fin_data <- function() { } get_and_clean_debt_fin_data <- function() { - debt_fin_data_raw <- read_rds(paste0(analysis_inputs_path, "/debt_financial_data.rda")) + debt_fin_data_raw <- read_rds(file.path(analysis_inputs_path, "debt_financial_data.rda")) return(debt_fin_data_raw) } @@ -1415,7 +1415,7 @@ create_audit_chart <- function(audit_file, proc_input_path) { ) - ggsave(paste0(proc_input_path, "/AuditChart.png"), Chart_1, height = nrow_Legend, width = 8) + ggsave(file.path(proc_input_path, "AuditChart.png"), Chart_1, height = nrow_Legend, width = 8) } create_audit_file <- function(portfolio_total) { @@ -1548,7 +1548,7 @@ get_average_emission_data <- function(inc_emission_factors) { average_sector_intensity <- data.frame() if (inc_emission_factors) { - average_sector_intensity <- read_rda(paste0(analysis_inputs_path, "/average_sector_intensity.rda")) + average_sector_intensity <- read_rda(file.path(analysis_inputs_path, "average_sector_intensity.rda")) } return(average_sector_intensity) } @@ -1557,7 +1557,7 @@ get_company_emission_data <- function(inc_emission_factors) { company_emissions <- data.frame() if (inc_emission_factors) { - company_emissions <- read_rda(paste0(analysis_inputs_path, "/company_emissions.rda")) + company_emissions <- read_rda(file.path(analysis_inputs_path, "company_emissions.rda")) } return(company_emissions) } From 314d7e6b4f6bea9f182baaa893c0eaabee52ce76 Mon Sep 17 00:00:00 2001 From: Clare2D Date: Thu, 18 Feb 2021 09:47:04 +0100 Subject: [PATCH 08/12] Revert "replace paste0 with file.path" This reverts commit 495f06286a66794238985893db5e7c9e97283661. --- 0_portfolio_input_check_functions.R | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/0_portfolio_input_check_functions.R b/0_portfolio_input_check_functions.R index e7b7b88e..a6aa2f54 100644 --- a/0_portfolio_input_check_functions.R +++ b/0_portfolio_input_check_functions.R @@ -4,29 +4,29 @@ read_raw_portfolio_file <- function(project_name) { portfolio <- NA - input_path <- file.path(project_location, "20_Raw_Inputs") + input_path <- paste0(project_location, "/20_Raw_Inputs/") csv_to_read <- list.files(path = input_path, pattern = "_Input.csv") txt_to_read <- list.files(path = input_path, pattern = "_Input.txt") if (length(csv_to_read) == 1) { - portfolio <- read_csv(file.path(input_path, csv_to_read)) + portfolio <- read_csv(paste0(input_path, csv_to_read)) } if (length(txt_to_read) == 1) { - enc <- guess_encoding(file.path(input_path, txt_to_read))$encoding[1] - portfolio <- read.table(file.path(input_path, txt_to_read), sep = ",", header = T, fileEncoding = enc) + enc <- guess_encoding(paste0(input_path, txt_to_read))$encoding[1] + portfolio <- read.table(paste0(input_path, txt_to_read), sep = ",", header = T, fileEncoding = enc) } # Reads in Files saved with a ; not a , if (ncol(portfolio) == 1 & length(csv_to_read) == 1) { - portfolio <- read.csv(file.path(input_path, csv_to_read), strip.white = T, stringsAsFactors = F, sep = ";") + portfolio <- read.csv(paste0(input_path, csv_to_read), strip.white = T, stringsAsFactors = F, sep = ";") } if (ncol(portfolio) == 1 & length(txt_to_read) == 1) { - portfolio <- read.table(file.path(input_path, txt_to_read), sep = "\t", header = T, fileEncoding = enc) + portfolio <- read.table(paste0(input_path, txt_to_read), sep = "\t", header = T, fileEncoding = enc) } if (ncol(portfolio) == 1 & length(txt_to_read) == 1) { - portfolio <- read.table(file.path(input_path, txt_to_read), sep = ";", header = T, fileEncoding = enc) + portfolio <- read.table(paste0(input_path, txt_to_read), sep = ";", header = T, fileEncoding = enc) } @@ -896,13 +896,13 @@ get_and_clean_currency_data <- function() { get_and_clean_fund_data <- function() { fund_data <- NA # Fund Data - if (file.exists(file.path(analysis_inputs_path, "fund_data_", financial_timestamp, ".rda"))) { - fund_data <- readRDS(file.path(analysis_inputs_path, "fund_data_", financial_timestamp, ".rda")) - } else if (file.exists(file.path(analysis_inputs_path, "fund_data_2018Q4.rda"))) { - fund_data <- readRDS(file.path(analysis_inputs_path, "fund_data_2018Q4.rda")) + if (file.exists(paste0(analysis_inputs_path, "/fund_data_", financial_timestamp, ".rda"))) { + fund_data <- readRDS(paste0(analysis_inputs_path, "/fund_data_", financial_timestamp, ".rda")) + } else if (file.exists(paste0(analysis_inputs_path, "/fund_data_2018Q4.rda"))) { + fund_data <- readRDS(paste0(analysis_inputs_path, "/fund_data_2018Q4.rda")) print("Old Fund Data being used. Replace FundsData2018Q4 or check name of file.") - } else if (file.exists(file.path(analysis_inputs_path, "SFC_26052020_funds.csv"))) { - fund_data <- read_csv(file.path(analysis_inputs_path, "SFC_26052020_funds.csv")) + } else if (file.exists(paste0(analysis_inputs_path, "/SFC_26052020_funds.csv"))) { + fund_data <- read_csv(paste0(analysis_inputs_path, "/SFC_26052020_funds.csv")) print("2020Q2 SFC fund data being used") } else { if (!data_check(fund_data)) { @@ -923,7 +923,7 @@ get_and_clean_fund_data <- function() { get_and_clean_fin_data <- function(fund_data) { # Financial Data - fin_data_raw <- read_rda(file.path(analysis_inputs_path, "security_financial_data.rda")) %>% as_tibble() + fin_data_raw <- read_rda(paste0(analysis_inputs_path, "/security_financial_data.rda")) %>% as_tibble() # remove unclear duplicates from raw financial data. This should be moved to DataStore. rm_duplicates <- read_csv("non_distinct_isins.csv") @@ -1011,7 +1011,7 @@ get_and_clean_revenue_data <- function() { revenue_data <- data.frame() if (has_revenue) { - revenue_data <- read_rda(file.path(analysis_inputs_path, "revenue_data_member_ticker.rda")) + revenue_data <- read_rda(paste0(analysis_inputs_path, "/revenue_data_member_ticker.rda")) # col_types = "dcdcclcd") revenue_data <- revenue_data %>% @@ -1024,7 +1024,7 @@ get_and_clean_revenue_data <- function() { } get_and_clean_company_fin_data <- function() { - comp_fin_data_raw <- read_rds(file.path(analysis_inputs_path, "consolidated_financial_data.rda")) + comp_fin_data_raw <- read_rds(paste0(analysis_inputs_path, "/consolidated_financial_data.rda")) comp_fin_data_raw <- comp_fin_data_raw %>% select( company_id, company_name, bloomberg_id, country_of_domicile, corporate_bond_ticker, bics_subgroup, bics_sector, @@ -1040,7 +1040,7 @@ get_and_clean_company_fin_data <- function() { } get_and_clean_debt_fin_data <- function() { - debt_fin_data_raw <- read_rds(file.path(analysis_inputs_path, "debt_financial_data.rda")) + debt_fin_data_raw <- read_rds(paste0(analysis_inputs_path, "/debt_financial_data.rda")) return(debt_fin_data_raw) } @@ -1415,7 +1415,7 @@ create_audit_chart <- function(audit_file, proc_input_path) { ) - ggsave(file.path(proc_input_path, "AuditChart.png"), Chart_1, height = nrow_Legend, width = 8) + ggsave(paste0(proc_input_path, "/AuditChart.png"), Chart_1, height = nrow_Legend, width = 8) } create_audit_file <- function(portfolio_total) { @@ -1548,7 +1548,7 @@ get_average_emission_data <- function(inc_emission_factors) { average_sector_intensity <- data.frame() if (inc_emission_factors) { - average_sector_intensity <- read_rda(file.path(analysis_inputs_path, "average_sector_intensity.rda")) + average_sector_intensity <- read_rda(paste0(analysis_inputs_path, "/average_sector_intensity.rda")) } return(average_sector_intensity) } @@ -1557,7 +1557,7 @@ get_company_emission_data <- function(inc_emission_factors) { company_emissions <- data.frame() if (inc_emission_factors) { - company_emissions <- read_rda(file.path(analysis_inputs_path, "company_emissions.rda")) + company_emissions <- read_rda(paste0(analysis_inputs_path, "/company_emissions.rda")) } return(company_emissions) } From 08e9c767922823639bf36db83fd346029c888446 Mon Sep 17 00:00:00 2001 From: Mauro Lepore Date: Thu, 18 Feb 2021 12:48:56 -0600 Subject: [PATCH 09/12] Revert "ran styler on repo" This reverts commit 2d2a54e6e6c69fc95b1e23a85221cfeb689d04e2. --- 0_global_functions.R | 7 +- 0_portfolio_input_check_functions.R | 224 +++++++++++++--------------- 0_portfolio_test.R | 13 +- 0_web_functions.R | 57 ++++--- R/get_param.R | 1 + R/utils.R | 4 +- test_test_cases.R | 39 ++--- web_tool_script_1.R | 45 +++--- web_tool_script_2.R | 9 +- web_tool_script_3.R | 78 +++++----- 10 files changed, 218 insertions(+), 259 deletions(-) diff --git a/0_global_functions.R b/0_global_functions.R index a8d9224d..418e238b 100644 --- a/0_global_functions.R +++ b/0_global_functions.R @@ -43,7 +43,7 @@ set_col_types <- function(grouping_variables, fixed_col_types) { return(port_col_types) } -set_project_parameters <- function(file_path) { +set_project_parameters <- function(file_path){ cfg <- config::get(file = file_path) project_report_name <<- cfg$reporting$project_report_name @@ -124,6 +124,7 @@ set_project_parameters <- function(file_path) { inc_stresstest <<- FALSE warning("Warning: inc_stresstest set to standard value (FALSE) as it is not defined in the parameter file") } + } set_global_parameters <- function(file_path) { @@ -250,6 +251,7 @@ set_git_path <- function() { } set_analysis_inputs_path <- function(twodii_internal, data_location_ext, dataprep_ref = datastore_timestamp) { + if (twodii_internal) { analysis_inputs_path <- r2dii.utils::path_dropbox_2dii("PortCheck", "00_Data", "07_AnalysisInputs", dataprep_ref) analysis_inputs_path <- file.path(analysis_inputs_path) @@ -362,5 +364,6 @@ write_log <- function(msg, file_path = log_path, ...) { as.character(msg), ... ) - write(composed, file = file.path(file_path, "error_messages.txt"), append = TRUE) + write(composed, file = file.path(file_path,"error_messages.txt"), append = TRUE) } + diff --git a/0_portfolio_input_check_functions.R b/0_portfolio_input_check_functions.R index a6aa2f54..4c822db8 100644 --- a/0_portfolio_input_check_functions.R +++ b/0_portfolio_input_check_functions.R @@ -59,54 +59,44 @@ clean_portfolio_col_types <- function(portfolio, grouping_variables) { # portfolio[,grouping_variables] <- lapply(portfolio[,grouping_variables], clean_punctuation) if (is.character(portfolio$investor_name) == FALSE) { - write_log( - msg = paste0( - "Wrong variable class for investor_name. Should be character, but is ", - class(portfolio$investor_name), - ". This can introduce errors in further calculations!" - ), - file_path = log_path - ) + write_log(msg = paste0( + "Wrong variable class for investor_name. Should be character, but is ", + class(portfolio$investor_name), + ". This can introduce errors in further calculations!" + ), + file_path = log_path) } if (is.character(portfolio$portfolio_name) == FALSE) { - write_log( - msg = paste0( - "Wrong variable class for portfolio_name Should be character, but is ", - class(portfolio$portfolio_name), - ". This can introduce errors in further calculations!" - ), - file_path = log_path - ) + write_log(msg = paste0( + "Wrong variable class for portfolio_name Should be character, but is ", + class(portfolio$portfolio_name), + ". This can introduce errors in further calculations!" + ), + file_path = log_path) } if (is.numeric(portfolio$market_value) == FALSE) { - write_log( - msg = paste0( - "Wrong variable class for market_value Should be numeric, but is ", - class(portfolio$market_value), - ". This can introduce errors in further calculations!" - ), - file_path = log_path - ) + write_log(msg = paste0( + "Wrong variable class for market_value Should be numeric, but is ", + class(portfolio$market_value), + ". This can introduce errors in further calculations!" + ), + file_path = log_path) } if (is.character(portfolio$currency) == FALSE) { - write_log( - msg = paste0( - "Wrong variable class for currency Should be character, but is ", - class(portfolio$currency), - ". This can introduce errors in further calculations!" - ), - file_path = log_path - ) + write_log(msg = paste0( + "Wrong variable class for currency Should be character, but is ", + class(portfolio$currency), + ". This can introduce errors in further calculations!" + ), + file_path = log_path) } if (is.character(portfolio$isin) == FALSE) { - write_log( - msg = paste0( - "Wrong variable class for isin Should be character, but is ", - class(portfolio$isin), - ". This can introduce errors in further calculations!" - ), - file_path = log_path - ) + write_log(msg = paste0( + "Wrong variable class for isin Should be character, but is ", + class(portfolio$isin), + ". This can introduce errors in further calculations!" + ), + file_path = log_path) } ### what about number_of_shares??? @@ -122,15 +112,13 @@ clean_portfolio_col_types <- function(portfolio, grouping_variables) { clear_portfolio_input_blanks <- function(portfolio) { if (any(portfolio[, grouping_variables] == "" | is.na(portfolio[, grouping_variables]))) { print("Warning: missing grouping variables, corresponding rows removed") - write_log( - msg = paste( - "Warning: some entries of the uploaded portfolio file were removed + write_log(msg = paste( + "Warning: some entries of the uploaded portfolio file were removed because of missing values in at least one of the variables", str_c(grouping_variables, collapse = ", "), - "\n To ensure complete analysis, please upload a file without + "\n To ensure complete analysis, please upload a file without missing values in these columns." - ), - file_path = log_path - ) + ), + file_path = log_path) portfolio <- portfolio %>% filter_at( grouping_variables, all_vars(!is.na(.)) @@ -242,18 +230,18 @@ map_security_sectors <- function(fin_data, sector_bridge) { fin_data <- fin_data %>% left_join(sector_bridge %>% filter(source == "BICS") %>% select(-source), - by = c("security_bics_subgroup" = "industry_classification") + by = c("security_bics_subgroup" = "industry_classification") ) %>% mutate(security_icb_subsector = as.character(security_icb_subsector)) fin_data_na <- fin_data %>% filter(is.na(sector)) %>% - select(-c(sector, sector_boe, subsector_boe, sector_dnb, sector_ipr, subsector_ipr)) + select(-c(sector,sector_boe,subsector_boe,sector_dnb,sector_ipr,subsector_ipr)) fin_data <- fin_data %>% filter(!is.na(sector)) fin_data_na <- fin_data_na %>% left_join(sector_bridge %>% filter(source == "ICB") %>% select(-source), - by = c("security_icb_subsector" = "industry_classification") + by = c("security_icb_subsector" = "industry_classification") ) fin_data <- fin_data %>% bind_rows(fin_data_na) @@ -279,7 +267,7 @@ map_comp_sectors <- function(comp_fin_data, sector_bridge) { initial_no_rows <- nrow(comp_fin_data) comp_fin_data <- comp_fin_data %>% left_join(sector_bridge %>% filter(source == "BICS") %>% select(-source), - by = c("bics_subgroup" = "industry_classification") + by = c("bics_subgroup" = "industry_classification") ) comp_fin_data_na <- comp_fin_data %>% @@ -289,7 +277,7 @@ map_comp_sectors <- function(comp_fin_data, sector_bridge) { comp_fin_data <- comp_fin_data %>% filter(!is.na(sector)) comp_fin_data_na <- comp_fin_data_na %>% left_join(sector_bridge %>% filter(source == "ICB") %>% select(-source), - by = c("icb_subgroup" = "industry_classification") + by = c("icb_subgroup" = "industry_classification") ) comp_fin_data <- comp_fin_data %>% bind_rows(comp_fin_data_na) @@ -886,9 +874,7 @@ get_and_clean_currency_data <- function() { currencies <- set_currency_timestamp(currencies) - if (all(currencies$exchange_rate) == 1) { - print("currency data temporary. todo: update") - } + if (all(currencies$exchange_rate) == 1){print("currency data temporary. todo: update")} currencies } @@ -938,7 +924,7 @@ get_and_clean_fin_data <- function(fund_data) { } overrides <- read_csv("data/fin_sector_overrides.csv", - col_types = "ccdc" + col_types = "ccdc" ) sector_bridge <- read_csv("data/sector_bridge.csv", col_types = "cccccccc") @@ -1050,7 +1036,7 @@ process_raw_portfolio <- function(portfolio_raw, fund_data, currencies, grouping_variables, - total_fund_list = NA) { + total_fund_list=NA) { portfolio <- clean_colnames_portfolio_input_file(portfolio_raw) portfolio <- clear_portfolio_input_blanks(portfolio) @@ -1086,9 +1072,7 @@ process_raw_portfolio <- function(portfolio_raw, original_value_usd <- sum(portfolio$value_usd, na.rm = T) # correct Funds classification by comparing isin to the list of all known funds isins - if (!is.na(total_fund_list)) { - portfolio <- portfolio %>% mutate(asset_type = ifelse(is.element(isin, total_fund_list$fund_isin), "Funds", asset_type)) - } + if(!is.na(total_fund_list)){portfolio <- portfolio %>% mutate(asset_type = ifelse(is.element(isin, total_fund_list$fund_isin), "Funds", asset_type))} # identify funds in the portfolio fund_portfolio <- identify_fund_portfolio(portfolio) @@ -1183,19 +1167,19 @@ get_fund_coverage <- function(portfolio_raw, fund_portfolio_total <- portfolio_total %>% filter(!is.na(fund_isin)) - fund_portfolio_total_mapped_value_usd <- fund_portfolio_total %>% + fund_portfolio_total_mapped_value_usd <- fund_portfolio_total %>% group_by(holding_id) %>% summarize(total_mapped_value_usd = sum(value_usd)) - fund_portfolio_missing_value_usd <- fund_portfolio_total %>% - filter(nchar(isin) != 12) %>% + fund_portfolio_missing_value_usd <- fund_portfolio_total %>% + filter(nchar(isin)!=12) %>% group_by(holding_id) %>% summarize(missing_value_usd = sum(value_usd)) - fund_portfolio_funds_in_funds_not_mapped_value_usd <- fund_portfolio_total %>% - filter(nchar(isin) == 12 & asset_type == "Funds") %>% + fund_portfolio_funds_in_funds_not_mapped_value_usd <- fund_portfolio_total %>% + filter(nchar(isin)==12 & asset_type == "Funds") %>% group_by(holding_id) %>% summarize(funds_in_funds_not_mapped = sum(value_usd)) @@ -1217,7 +1201,7 @@ get_fund_coverage <- function(portfolio_raw, fund_portfolio <- fund_portfolio %>% mutate(effective_coverage = (total_mapped_value_usd - missing_value_usd - funds_in_funds_not_mapped) / value_usd) fund_portfolio <- fund_portfolio %>% mutate(fund_data_file_coverage = (total_mapped_value_usd) / value_usd) fund_portfolio <- fund_portfolio %>% mutate(lipper_data_coverage = (total_mapped_value_usd - missing_value_usd) / total_mapped_value_usd) - fund_portfolio <- fund_portfolio %>% mutate(lost_coverage_fif = 1 - (total_mapped_value_usd - missing_value_usd - funds_in_funds_not_mapped) / (total_mapped_value_usd - missing_value_usd)) + fund_portfolio <- fund_portfolio %>% mutate(lost_coverage_fif = 1 - (total_mapped_value_usd - missing_value_usd - funds_in_funds_not_mapped) / (total_mapped_value_usd-missing_value_usd)) @@ -1227,33 +1211,30 @@ get_fund_coverage <- function(portfolio_raw, } -summarize_fund_coverage <- function(fund_portfolio) { - fund_portfolio %>% select( - investor_name, - portfolio_name, - isin, - value_usd, - company_name, - effective_coverage - ) +summarize_fund_coverage <- function(fund_portfolio) { + + fund_portfolio %>% select(investor_name, + portfolio_name, + isin, + value_usd, + company_name, + effective_coverage) return(fund_portfolio) } -list_unknown_funds_in_funds <- function(portfolio_total) { +list_unknown_funds_in_funds <- function(portfolio_total) { + fund_portfolio_total <- portfolio_total %>% filter(!is.na(fund_isin)) table_of_funds_in_funds_not_mapped <- fund_portfolio_total %>% - filter(nchar(isin) == 12 & asset_type == "Funds") %>% - mutate(direct_holding = "FALSE") %>% - select( - investor_name, - portfolio_name, - isin, - value_usd, - company_name, - fund_isin - ) %>% - rename(parent_fund_isin = fund_isin) + filter(nchar(isin)==12 & asset_type == "Funds") %>% + mutate(direct_holding="FALSE") %>% + select(investor_name, + portfolio_name, + isin, + value_usd, + company_name, + fund_isin) %>% rename(parent_fund_isin = fund_isin) return(table_of_funds_in_funds_not_mapped) } @@ -1309,6 +1290,7 @@ create_merged_portfolio <- function(eq_portfolio, cb_portfolio) { } create_portfolio_subset <- function(portfolio, portfolio_type) { + if (portfolio_type %in% unique(portfolio$asset_type)) { portfolio_subset <- portfolio %>% ungroup() %>% @@ -1359,7 +1341,7 @@ create_audit_chart <- function(audit_file, proc_input_path) { flag_categories <- names(flag_numbers_colors) flag_numbers$flag <- factor(flag_numbers$flag, - levels = flag_categories + levels = flag_categories ) y_intercept <- sum(flag_numbers$perc_n[flag_numbers$flag != "Included in analysis"]) @@ -1367,13 +1349,13 @@ create_audit_chart <- function(audit_file, proc_input_path) { nrows <- length(unique(flag_numbers$flag)) nrow_Legend <- ifelse(nrows <= 2, 3.2, - ifelse(nrows == 3 | nrows == 4, 3.7, - ifelse(nrows == 5, 3.8, - ifelse(nrows == 6, 4.0, - ifelse(nrows == 7, 4, 4.6) - ) - ) - ) + ifelse(nrows == 3 | nrows == 4, 3.7, + ifelse(nrows == 5, 3.8, + ifelse(nrows == 6, 4.0, + ifelse(nrows == 7, 4, 4.6) + ) + ) + ) ) ## Chart @@ -1408,10 +1390,10 @@ create_audit_chart <- function(audit_file, proc_input_path) { guides(fill = guide_legend(reverse = TRUE, ncol = 1)) + geom_hline(aes(yintercept = y_intercept), color = "#3D3D3C", size = 1, show.legend = FALSE) + geom_text(aes(label = "Not included\nin analysis\n\n", x = 1.5, y_intercept - 0.02), - color = "#2e2e2e", family = base_family, size = base_size / 2.5, hjust = 1 + color = "#2e2e2e", family = base_family, size = base_size / 2.5, hjust = 1 ) + geom_text(aes(label = "Included\nin analysis\n\n", x = 1.5, y_intercept + 0.02), - color = "#207ddb", family = base_family, size = base_size / 2.5, hjust = 0 + color = "#207ddb", family = base_family, size = base_size / 2.5, hjust = 0 ) @@ -1461,6 +1443,7 @@ clean_unmatched_holdings <- function(portfolio) { calculate_average_portfolio_emissions <- function(portfolio_total, comp_fin_data, average_sector_intensity) { + min_portfolio <- portfolio_total %>% select( investor_name, @@ -1483,7 +1466,7 @@ calculate_average_portfolio_emissions <- function(portfolio_total, financial_sector, bics_sector ) %>% - summarise(value_usd = sum(value_usd, na.rm = T), .groups = "drop") %>% + summarise(value_usd = sum(value_usd, na.rm = T), .groups = "drop") %>% left_join( select( average_sector_intensity, @@ -1515,7 +1498,7 @@ calculate_average_portfolio_emissions <- function(portfolio_total, mean_intensity.x ) ) %>% - select(-mean_intensity.x, -mean_intensity.y) %>% + select(-mean_intensity.x, -mean_intensity.y) %>% mutate(weighted_sector_emissions = value_usd * mean_intensity) min_portfolio <- min_portfolio %>% @@ -1531,7 +1514,7 @@ calculate_average_portfolio_emissions <- function(portfolio_total, sector ) ) %>% - group_by(investor_name, portfolio_name, asset_type, sector) %>% + group_by(investor_name, portfolio_name, asset_type,sector) %>% summarise( weighted_sector_emissions = sum(weighted_sector_emissions, na.rm = T), .groups = "drop" @@ -1563,11 +1546,11 @@ get_company_emission_data <- function(inc_emission_factors) { } prepare_portfolio_emissions <- function( - audit_file, - fin_data, - comp_fin_data, - average_sector_intensity, - company_emissions) { + audit_file, + fin_data, + comp_fin_data, + average_sector_intensity, + company_emissions) { audit_file <- audit_file %>% janitor::clean_names(case = "snake") @@ -1731,12 +1714,12 @@ prepare_portfolio_emissions <- function( } calculate_portfolio_emissions <- function( - inc_emission_factors, - audit_file, - fin_data, - comp_fin_data, - average_sector_intensity, - company_emissions) { + inc_emission_factors, + audit_file, + fin_data, + comp_fin_data, + average_sector_intensity, + company_emissions) { audit_sector_emissions <- data.frame() if (inc_emission_factors) { @@ -1798,19 +1781,24 @@ add_other_to_sector_classifications <- function(audit) { } -pw_calculations <- function(eq_portfolio, cb_portfolio) { +pw_calculations <- function(eq_portfolio, cb_portfolio){ + port_all <- data.frame() - if (data_check(eq_portfolio)) { + if(data_check(eq_portfolio)){ + port_all <- bind_rows(port_all, eq_portfolio) } - if (data_check(cb_portfolio)) { + if(data_check(cb_portfolio)){ + port_all <- bind_rows(port_all, cb_portfolio) + } - if (data_check(port_all)) { - port_all <- port_all %>% select(!!!rlang::syms(grouping_variables), company_id, value_usd) + if(data_check(port_all)){ + + port_all <- port_all %>% select(!!!rlang::syms(grouping_variables),company_id, value_usd) port_all <- calculate_port_weight(port_all, grouping_variables) @@ -1820,9 +1808,11 @@ pw_calculations <- function(eq_portfolio, cb_portfolio) { summarise(port_weight = sum(port_weight, na.rm = T), .groups = "drop") %>% select(company_id, port_weight) %>% rename(portfolio_weight = port_weight) - } else { + + }else{ pw <- data.frame(company_id = "No companies in portfolio", portfolio_weight = "0") } return(pw) + } diff --git a/0_portfolio_test.R b/0_portfolio_test.R index 89eaad8f..34db0071 100644 --- a/0_portfolio_test.R +++ b/0_portfolio_test.R @@ -81,6 +81,7 @@ calculate_ownership_weight <- function(portfolio) { } calculate_port_weight <- function(portfolio, grouping_variables) { + portfolio <- portfolio %>% ungroup() %>% group_by(!!!rlang::syms(grouping_variables)) %>% @@ -166,10 +167,8 @@ aggregate_portfolio <- function(df) { plan_tech_prod, plan_alloc_wt_tech_prod, plan_carsten, plan_emission_factor, scen_tech_prod, scen_alloc_wt_tech_prod, scen_carsten, scen_emission_factor ) %>% - mutate( - plan_emission_factor = ifelse(is.na(plan_emission_factor), 0, plan_emission_factor), - scen_emission_factor = ifelse(is.na(scen_emission_factor), 0, scen_emission_factor) - ) %>% + mutate(plan_emission_factor = ifelse(is.na(plan_emission_factor), 0, plan_emission_factor), + scen_emission_factor = ifelse(is.na(scen_emission_factor), 0, scen_emission_factor)) %>% group_by( !!!rlang::syms(grouping_variables), scenario, allocation, equity_market, scenario_geography, year, @@ -212,10 +211,8 @@ aggregate_map_data <- function(portfolio) { ald_sector, technology, financial_sector, allocation, allocation_weight, ald_production_unit ) %>% - summarise( - plan_alloc_wt_tech_prod = sum(plan_alloc_wt_tech_prod, na.rm = TRUE), - .groups = "drop_last" - ) %>% + summarise(plan_alloc_wt_tech_prod = sum(plan_alloc_wt_tech_prod, na.rm = TRUE), + .groups = "drop_last") %>% mutate(plan_alloc_wt_sec_prod = sum(plan_alloc_wt_tech_prod)) if (data_check(portfolio)) { diff --git a/0_web_functions.R b/0_web_functions.R index 7b9ef50f..1e830d76 100644 --- a/0_web_functions.R +++ b/0_web_functions.R @@ -106,6 +106,7 @@ add_naming_to_portfolio <- function(portfolio_raw) { } get_input_files <- function(portfolio_name_ref_all) { + portfolio <- tibble() input_path <- file.path(project_location, "20_Raw_Inputs") @@ -220,14 +221,14 @@ read_web_input_file <- function(input_file_path) { if (data_check(input_file) == FALSE) { warning("Input file not readable") ifelse(nrow(input_file) == 0, - write_log( - msg = "Input file has 0 rows. Please ensure the uploaded file is not empty.", - file_path = log_path - ), - write_log( - msg = "Input file could not be transformed into a data.frame. Please check the uploaded file has the correct format.", - file_path = log_path - ) + write_log( + msg = "Input file has 0 rows. Please ensure the uploaded file is not empty.", + file_path = log_path + ), + write_log( + msg = "Input file could not be transformed into a data.frame. Please check the uploaded file has the correct format.", + file_path = log_path + ) ) } @@ -305,7 +306,7 @@ save_cleaned_files <- function(save_loc, debt_fin_data, average_sector_intensity, company_emissions, - total_fund_list = NA) { + total_fund_list=NA) { if (!dir.exists(save_loc)) { dir.create(save_loc) } @@ -317,9 +318,8 @@ save_cleaned_files <- function(save_loc, fst::write_fst(debt_fin_data, file.path(save_loc, "debt_fin_data.fst")) fst::write_fst(average_sector_intensity, file.path(save_loc, "average_sector_intensity.fst")) fst::write_fst(company_emissions, file.path(save_loc, "company_emissions.fst")) - if (!is.na(total_fund_list)) { - fst::write_fst(total_fund_list, file.path(save_loc, "total_fund_list.fst")) - } + if (!is.na(total_fund_list)) + {fst::write_fst(total_fund_list, file.path(save_loc, "total_fund_list.fst"))} if (check_file_size(save_loc)) warning("File size exceeds what can be pushed to GitHub. Check before Committing") } @@ -329,7 +329,7 @@ check_file_size <- function(folder_to_check) { any(file.size(files_to_check) > 100e6) } -empty_portfolio_results <- function() { +empty_portfolio_results <- function(){ tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "scenario" = NA_character_, "allocation" = NA_character_, @@ -348,7 +348,7 @@ empty_portfolio_results <- function() { ) } -empty_company_results <- function() { +empty_company_results <- function(){ tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "scenario" = NA_character_, "allocation" = NA_character_, @@ -371,23 +371,19 @@ empty_company_results <- function() { ) } -empty_emissions_results <- function() { - tibble( - "investor_name" = NA_character_, "portfolio_name" = NA_character_, - "asset_type" = NA_character_, "sector" = NA_character_, - "weighted_sector_emissions" = NA_real_ - ) +empty_emissions_results <- function(){ + tibble("investor_name" = NA_character_, "portfolio_name" = NA_character_, + "asset_type" = NA_character_, "sector" = NA_character_, + "weighted_sector_emissions" = NA_real_) } -empty_audit_file <- function() { - tibble( - "investor_name" = NA_character_, "portfolio_name" = NA_character_, - "asset_type" = NA_character_, "valid_input" = NA, "isin" = NA_character_, - "direct_holding" = NA, "value_usd" = NA_real_ - ) +empty_audit_file <- function(){ + tibble("investor_name" = NA_character_, "portfolio_name" = NA_character_, + "asset_type" = NA_character_, "valid_input" = NA, "isin" = NA_character_, + "direct_holding" = NA, "value_usd" = NA_real_) } -empty_map_results <- function() { +empty_map_results <- function(){ tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "ald_location" = NA_character_, "year" = NA_integer_, @@ -400,7 +396,7 @@ empty_map_results <- function() { ) } -empty_st_results <- function() { +empty_st_results <- function(){ tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "ald_sector" = NA_character_, "technology" = NA_character_, @@ -414,7 +410,7 @@ empty_st_results <- function() { ) } -empty_ipr_st_results <- function() { +empty_ipr_st_results <- function(){ tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "sector" = NA_character_, "subsector" = NA_character_, @@ -424,7 +420,7 @@ empty_ipr_st_results <- function() { ) } -empty_portfolio_overview <- function() { +empty_portfolio_overview <- function(){ tibble( "investor_name" = NA_character_, "portfolio_name" = NA_character_, "asset_type" = NA_character_, "financial_sector" = NA_character_, @@ -432,3 +428,4 @@ empty_portfolio_overview <- function() { "asset_value_usd" = NA_real_, "portfolio_value_usd" = NA_real_ ) } + diff --git a/R/get_param.R b/R/get_param.R index f6bf3831..65bd5b67 100644 --- a/R/get_param.R +++ b/R/get_param.R @@ -36,3 +36,4 @@ pluck_param <- function(x, file, if_null) { out } + diff --git a/R/utils.R b/R/utils.R index 2caa813f..8432f5f3 100644 --- a/R/utils.R +++ b/R/utils.R @@ -76,7 +76,7 @@ dockerfile_packages <- function(path = packages_path()) { c( ' && Rscript -e "install.packages( \\', - paste0(" ", format_as_vector(pkg), " \\"), + paste0(' ', format_as_vector(pkg), ' \\'), ' )" \\' ) } @@ -84,7 +84,7 @@ dockerfile_packages <- function(path = packages_path()) { format_as_vector <- function(string) { x <- glue("'{string}',") x[length(x)] <- sub(",$", "", x[length(x)]) - c("c(", glue(" {x}"), ")") + c('c(', glue(" {x}"), ')' ) } packages_path <- function() { diff --git a/test_test_cases.R b/test_test_cases.R index b21e0cd6..014f2f82 100644 --- a/test_test_cases.R +++ b/test_test_cases.R @@ -20,12 +20,8 @@ for (i in seq_along(test_cases_csvs)) { investor_name <- unique(test_case$Investor.Name) # if no unique, valid portfolio_name or investor_name, use the filename - if (length(portfolio_name) != 1) { - portfolio_name <- filename - } - if (length(investor_name) != 1) { - investor_name <- filename - } + if (length(portfolio_name) != 1) { portfolio_name <- filename } + if (length(investor_name) != 1) { investor_name <- filename } yaml_data <- list(default = list( @@ -45,10 +41,8 @@ for (i in seq_along(test_cases_csvs)) { fs::dir_create(out_dir, recurse = TRUE) sub_directories_needed <- - c( - "00_Log_Files", "10_Parameter_File", "20_Raw_Inputs", - "30_Processed_Inputs", "40_Results", "50_Outputs" - ) + c("00_Log_Files", "10_Parameter_File", "20_Raw_Inputs", + "30_Processed_Inputs", "40_Results", "50_Outputs") lapply(sub_directories_needed, function(sub_dir) { fs::dir_create(fs::path(out_dir, sub_dir), recurse = TRUE) @@ -56,16 +50,14 @@ for (i in seq_along(test_cases_csvs)) { write_csv(test_case, fs::path(out_dir, "20_Raw_Inputs", paste0(portfolio_name, ".csv"))) write_yaml(yaml_data, fs::path(out_dir, "10_Parameter_File", paste0(portfolio_name, "_PortfolioParameters.yml")), indent = 4) - fs::file_copy( - "working_dir/10_Parameter_File/AnalysisParameters.yml", - fs::path(out_dir, "10_Parameter_File", "AnalysisParameters.yml") - ) + fs::file_copy("working_dir/10_Parameter_File/AnalysisParameters.yml", + fs::path(out_dir, "10_Parameter_File", "AnalysisParameters.yml")) } report_git_status <- function(repo_roots = ".") { - for (repo_root in repo_roots) { + for (repo_root in repo_roots) { cli::cli_h1(paste0("repo status for: ", repo_root)) system2("git", c("-C", repo_root, "fetch"), stdout = FALSE) @@ -91,10 +83,8 @@ report_git_status <- owner <- "2DegreesInvesting" repo_name <- basename(repo_root) open_pr_list <- - gh::gh("/repos/:owner/:repo/pulls", - owner = owner, repo = repo_name, - state = "open", .limit = Inf - ) + gh::gh("/repos/:owner/:repo/pulls", owner = owner, repo = repo_name, + state = "open", .limit = Inf) if (length(open_pr_list) > 0) { infos <- @@ -107,8 +97,7 @@ report_git_status <- } report_git_status( - c( - "../PACTA_analysis", + c("../PACTA_analysis", "../create_interactive_report", "../StressTestingModelDev", "../pacta-data", @@ -125,14 +114,10 @@ for (csv_num in seq_along(test_cases_csvs)) { sub_directory <- portfolio_name # if no unique, valid portfolio_name, use an empty string - if (length(portfolio_name) != 1) { - portfolio_name <- "" - } + if (length(portfolio_name) != 1) { portfolio_name <- "" } # if no unique, valid portfolio_name, use the filename - if (length(sub_directory) != 1) { - sub_directory <- filename - } + if (length(sub_directory) != 1) { sub_directory <- filename } out_dir <- fs::path(test_cases_output_dir, sub_directory) diff --git a/web_tool_script_1.R b/web_tool_script_1.R index dd065f79..e606ba28 100644 --- a/web_tool_script_1.R +++ b/web_tool_script_1.R @@ -9,12 +9,8 @@ source("0_web_functions.R") source("0_json_functions.R") source("0_portfolio_test.R") -if (!exists("portfolio_name_ref_all")) { - portfolio_name_ref_all <- "TestPortfolio_Input" -} -if (!exists("portfolio_root_dir")) { - portfolio_root_dir <- "working_dir" -} +if (!exists("portfolio_name_ref_all")) { portfolio_name_ref_all <- "TestPortfolio_Input" } +if (!exists("portfolio_root_dir")) { portfolio_root_dir <- "working_dir" } portfolio_root_dir <- "working_dir" setup_project() @@ -25,7 +21,7 @@ set_webtool_paths(portfolio_root_dir) set_portfolio_parameters(file_path = fs::path(par_file_path, paste0(portfolio_name_ref_all, "_PortfolioParameters.yml"))) -set_project_parameters(file.path(working_location, "parameter_files", paste0("ProjectParameters_", project_code, ".yml"))) +set_project_parameters(file.path(working_location, "parameter_files",paste0("ProjectParameters_", project_code, ".yml"))) # need to define an alternative location for data files analysis_inputs_path <- set_analysis_inputs_path(twodii_internal, data_location_ext, dataprep_timestamp) @@ -47,11 +43,11 @@ file_location <- file.path(analysis_inputs_path, "cleaned_files") if (new_data == TRUE) { currencies <- get_and_clean_currency_data() - - + + total_fund_list <- get_and_clean_total_fund_list_data() - - # fund_data <- get_and_clean_fund_data() + + # fund_data <- get_and_clean_fund_data() fund_data <- data.frame() fin_data <- get_and_clean_fin_data(fund_data) @@ -75,11 +71,11 @@ if (new_data == TRUE) { debt_fin_data, average_sector_intensity, company_emissions, - total_fund_list = total_fund_list + total_fund_list=total_fund_list ) } else { currencies <- fst::read_fst(file.path(file_location, "currencies.fst")) - + read_fst_or_return_null <- function(fst_file) { if (!file.exists(fst_file)) { return(NULL) @@ -90,19 +86,19 @@ if (new_data == TRUE) { fund_data_path <- file.path(file_location, "fund_data.fst") - + fund_data <- read_fst_or_return_null(fund_data_path) - + fund_data$holding_isin <- as.character(fund_data$holding_isin) fund_data$fund_isin <- as.character(fund_data$fund_isin) - - + + fin_data <- fst::read_fst(file.path(file_location, "fin_data.fst")) comp_fin_data <- fst::read_fst(file.path(file_location, "comp_fin_data.fst")) debt_fin_data <- fst::read_fst(file.path(file_location, "debt_fin_data.fst")) - + total_fund_list <- fst::read_fst(file.path(file_location, "total_fund_list.fst")) if (inc_emission_factors) { @@ -129,7 +125,7 @@ portfolio <- process_raw_portfolio( fund_data, currencies, grouping_variables, - total_fund_list = total_fund_list + total_fund_list=total_fund_list ) # information of coverage and coverage loses for all funds in raw_portfolio @@ -172,8 +168,7 @@ audit_file <- create_audit_file(portfolio_total) emissions_totals <- calculate_average_portfolio_emissions( portfolio_total, comp_fin_data, - average_sector_intensity -) + average_sector_intensity) port_weights <- pw_calculations(eq_portfolio, cb_portfolio) @@ -199,10 +194,10 @@ save_if_exists(emissions_totals, portfolio_name, file.path(proc_input_path_, "em save_if_exists(fund_coverage_summary, portfolio_name, file.path(proc_input_path_, "fund_coverage_summary.rda")) save_if_exists(unknown_funds_in_funds, portfolio_name, file.path(proc_input_path_, "unknown_funds_in_funds.rda")) -if (data_check(port_weights)) { - port_weights <- jsonlite::toJSON(x = port_weights) - write(x = port_weights, file = file.path(proc_input_path_, "portfolio_weights.json")) -} +if(data_check(port_weights)){ + port_weights <- jsonlite::toJSON(x=port_weights) + write(x = port_weights, file = file.path(proc_input_path_,"portfolio_weights.json")) + } rm(portfolio_total) rm(portfolio) diff --git a/web_tool_script_2.R b/web_tool_script_2.R index b9f4e076..5710d74b 100644 --- a/web_tool_script_2.R +++ b/web_tool_script_2.R @@ -11,12 +11,8 @@ source("0_portfolio_test.R") source("0_global_functions.R") source("0_web_functions.R") -if (!exists("portfolio_name_ref_all")) { - portfolio_name_ref_all <- "TestPortfolio_Input" -} -if (!exists("portfolio_root_dir")) { - portfolio_root_dir <- "working_dir" -} +if (!exists("portfolio_name_ref_all")) { portfolio_name_ref_all <- "TestPortfolio_Input" } +if (!exists("portfolio_root_dir")) { portfolio_root_dir <- "working_dir" } setup_project() @@ -134,6 +130,7 @@ bonds_inputs_file <- file.path(proc_input_path, portfolio_name_ref_all, "bonds_p # portfolio_name <- file_names$portfolio_name if (file.exists(bonds_inputs_file)) { + ald_scen_cb <- get_ald_scen("Bonds") ald_raw_cb <- get_ald_raw("Bonds") diff --git a/web_tool_script_3.R b/web_tool_script_3.R index 8d11291d..32a3cafb 100644 --- a/web_tool_script_3.R +++ b/web_tool_script_3.R @@ -6,12 +6,8 @@ use_r_packages() source("0_global_functions.R") source("0_web_functions.R") -if (!exists("portfolio_name_ref_all")) { - portfolio_name_ref_all <- "TestPortfolio_Input" -} -if (!exists("portfolio_root_dir")) { - portfolio_root_dir <- "working_dir" -} +if (!exists("portfolio_name_ref_all")) { portfolio_name_ref_all <- "TestPortfolio_Input" } +if (!exists("portfolio_root_dir")) { portfolio_root_dir <- "working_dir" } setup_project() @@ -19,14 +15,15 @@ set_webtool_paths(portfolio_root_dir) set_portfolio_parameters(file_path = fs::path(par_file_path, paste0(portfolio_name_ref_all, "_PortfolioParameters.yml"))) -set_project_parameters(file.path(working_location, "parameter_files", paste0("ProjectParameters_", project_code, ".yml"))) +set_project_parameters(file.path(working_location, "parameter_files",paste0("ProjectParameters_", project_code, ".yml"))) -if (project_code == "PA2020FL") { - peer_group <- case_when( - peer_group %in% c("other") ~ "Others", +if(project_code == "PA2020FL"){ + peer_group = case_when( + peer_group %in% c("other")~ "Others", peer_group %in% c("bank", "assetmanager") ~ "Banks and Asset Managers", peer_group %in% c("pensionfund", "insurance") ~ "Pension Funds and Insurances" ) + } @@ -36,28 +33,25 @@ source(file.path(template_path, "create_interactive_report.R")) source(file.path(template_path, "create_executive_summary.R")) source(file.path(template_path, "useful_functions.R")) -report_name <- select_report_template( - project_report_name = project_report_name, - language_select = language_select -) +report_name = select_report_template(project_report_name = project_report_name, + language_select = language_select) -exec_summary_name <- select_exec_summary_template( - project_report_name = project_report_name, - language_select = language_select -) +exec_summary_name = select_exec_summary_template(project_report_name = project_report_name, + language_select = language_select) -template_dir <- paste0(template_path, report_name, "/_book/") -exec_summary_dir <- paste0(template_path, exec_summary_name, "/") +template_dir <- paste0(template_path, report_name,"/_book/") +exec_summary_dir <- paste0(template_path, exec_summary_name,"/") survey_dir <- path(user_results_path, project_code, "survey") real_estate_dir <- path(user_results_path, project_code, "real_estate") output_dir <- file.path(outputs_path, portfolio_name_ref_all) -if (file.exists(file.path(proc_input_path, portfolio_name_ref_all, "audit_file.rda"))) { +if (file.exists(file.path(proc_input_path, portfolio_name_ref_all, "audit_file.rda"))){ audit_file <- readRDS(file.path(proc_input_path, portfolio_name_ref_all, "audit_file.rda")) -} else { +}else{ audit_file <- empty_audit_file() + } # load portfolio overview @@ -68,11 +62,10 @@ if (file.exists(file.path(proc_input_path, portfolio_name_ref_all, "overview_por } -if (file.exists(file.path(proc_input_path, portfolio_name_ref_all, "emissions.rda"))) { +if (file.exists(file.path(proc_input_path, portfolio_name_ref_all, "emissions.rda"))){ emissions <- read_rds(file.path(proc_input_path, portfolio_name_ref_all, "emissions.rda")) -} else { - emissions <- empty_emissions_results() -} +}else{ + emissions <- empty_emissions_results()} # load equity portfolio data if (file.exists(file.path(results_path, portfolio_name_ref_all, "Equity_results_portfolio.rda"))) { @@ -138,27 +131,27 @@ if (file.exists(file.path(results_path, portfolio_name_ref_all, "Stress_test_res } # load peers results both individual and aggregate -if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio.rda")))) { +if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio.rda")))){ peers_equity_results_portfolio <- read_rds(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio.rda"))) -} else { +}else{ peers_equity_results_portfolio <- empty_portfolio_results() } -if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio.rda")))) { +if(file.exists(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio.rda")))){ peers_bonds_results_portfolio <- read_rds(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio.rda"))) -} else { +}else{ peers_bonds_results_portfolio <- empty_portfolio_results() } -if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio_ind.rda")))) { +if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio_ind.rda")))){ peers_equity_results_user <- read_rds(file.path(data_location_ext, paste0(project_code, "_peers_equity_results_portfolio_ind.rda"))) -} else { +}else{ peers_equity_results_user <- empty_portfolio_results() } -if (file.exists(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio_ind.rda")))) { +if(file.exists(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio_ind.rda")))){ peers_bonds_results_user <- read_rds(file.path(data_location_ext, paste0(project_code, "_peers_bonds_results_portfolio_ind.rda"))) -} else { +}else{ peers_bonds_results_user <- empty_portfolio_results() } @@ -181,18 +174,18 @@ js_translations <- jsonlite::fromJSON( ) sector_order <- readr::read_csv( - path(template_path, "data", "sector_order", "sector_order.csv"), + path(template_path, "data","sector_order","sector_order.csv"), col_types = cols() ) # Needed for testing only shock <- shock_year # this should come directly from the stress test.. 2030 based on current discussions in CHPA2020 case -select_scenario_auto <- scenario_auto -select_scenario_other <- scenario_other -select_scenario_shipping <- scenario_shipping -twodi_sectors <- sector_list -repo_path <- template_path -file_name <- "template.Rmd" +select_scenario_auto = scenario_auto +select_scenario_other = scenario_other +select_scenario_shipping = scenario_shipping +twodi_sectors = sector_list +repo_path = template_path +file_name = "template.Rmd" create_interactive_report( repo_path = template_path, @@ -241,7 +234,7 @@ create_interactive_report( display_currency = display_currency, currency_exchange_value = currency_exchange_value, header_dictionary = header_dictionary, - sector_order = sector_order + sector_order = sector_order ) @@ -268,4 +261,5 @@ create_executive_summary( peers_bonds_results_portfolio = peers_bonds_results_portfolio, peers_equity_results_user = peers_equity_results_user, peers_bonds_results_user = peers_bonds_results_user + ) From 78688576ef34829d798cf3eaff12d67931a7da19 Mon Sep 17 00:00:00 2001 From: Clare2D Date: Thu, 18 Feb 2021 09:47:04 +0100 Subject: [PATCH 10/12] replace paste0 with file.path --- 0_portfolio_input_check_functions.R | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/0_portfolio_input_check_functions.R b/0_portfolio_input_check_functions.R index 4c822db8..2d9ceece 100644 --- a/0_portfolio_input_check_functions.R +++ b/0_portfolio_input_check_functions.R @@ -4,29 +4,29 @@ read_raw_portfolio_file <- function(project_name) { portfolio <- NA - input_path <- paste0(project_location, "/20_Raw_Inputs/") + input_path <- file.path(project_location, "20_Raw_Inputs") csv_to_read <- list.files(path = input_path, pattern = "_Input.csv") txt_to_read <- list.files(path = input_path, pattern = "_Input.txt") if (length(csv_to_read) == 1) { - portfolio <- read_csv(paste0(input_path, csv_to_read)) + portfolio <- read_csv(file.path(input_path, csv_to_read)) } if (length(txt_to_read) == 1) { - enc <- guess_encoding(paste0(input_path, txt_to_read))$encoding[1] - portfolio <- read.table(paste0(input_path, txt_to_read), sep = ",", header = T, fileEncoding = enc) + enc <- guess_encoding(file.path(input_path, txt_to_read))$encoding[1] + portfolio <- read.table(file.path(input_path, txt_to_read), sep = ",", header = T, fileEncoding = enc) } # Reads in Files saved with a ; not a , if (ncol(portfolio) == 1 & length(csv_to_read) == 1) { - portfolio <- read.csv(paste0(input_path, csv_to_read), strip.white = T, stringsAsFactors = F, sep = ";") + portfolio <- read.csv(file.path(input_path, csv_to_read), strip.white = T, stringsAsFactors = F, sep = ";") } if (ncol(portfolio) == 1 & length(txt_to_read) == 1) { - portfolio <- read.table(paste0(input_path, txt_to_read), sep = "\t", header = T, fileEncoding = enc) + portfolio <- read.table(file.path(input_path, txt_to_read), sep = "\t", header = T, fileEncoding = enc) } if (ncol(portfolio) == 1 & length(txt_to_read) == 1) { - portfolio <- read.table(paste0(input_path, txt_to_read), sep = ";", header = T, fileEncoding = enc) + portfolio <- read.table(file.path(input_path, txt_to_read), sep = ";", header = T, fileEncoding = enc) } @@ -882,13 +882,13 @@ get_and_clean_currency_data <- function() { get_and_clean_fund_data <- function() { fund_data <- NA # Fund Data - if (file.exists(paste0(analysis_inputs_path, "/fund_data_", financial_timestamp, ".rda"))) { - fund_data <- readRDS(paste0(analysis_inputs_path, "/fund_data_", financial_timestamp, ".rda")) - } else if (file.exists(paste0(analysis_inputs_path, "/fund_data_2018Q4.rda"))) { - fund_data <- readRDS(paste0(analysis_inputs_path, "/fund_data_2018Q4.rda")) + if (file.exists(file.path(analysis_inputs_path, "fund_data_", financial_timestamp, ".rda"))) { + fund_data <- readRDS(file.path(analysis_inputs_path, "fund_data_", financial_timestamp, ".rda")) + } else if (file.exists(file.path(analysis_inputs_path, "fund_data_2018Q4.rda"))) { + fund_data <- readRDS(file.path(analysis_inputs_path, "fund_data_2018Q4.rda")) print("Old Fund Data being used. Replace FundsData2018Q4 or check name of file.") - } else if (file.exists(paste0(analysis_inputs_path, "/SFC_26052020_funds.csv"))) { - fund_data <- read_csv(paste0(analysis_inputs_path, "/SFC_26052020_funds.csv")) + } else if (file.exists(file.path(analysis_inputs_path, "SFC_26052020_funds.csv"))) { + fund_data <- read_csv(file.path(analysis_inputs_path, "SFC_26052020_funds.csv")) print("2020Q2 SFC fund data being used") } else { if (!data_check(fund_data)) { @@ -909,7 +909,7 @@ get_and_clean_fund_data <- function() { get_and_clean_fin_data <- function(fund_data) { # Financial Data - fin_data_raw <- read_rda(paste0(analysis_inputs_path, "/security_financial_data.rda")) %>% as_tibble() + fin_data_raw <- read_rda(file.path(analysis_inputs_path, "security_financial_data.rda")) %>% as_tibble() # remove unclear duplicates from raw financial data. This should be moved to DataStore. rm_duplicates <- read_csv("non_distinct_isins.csv") @@ -997,7 +997,7 @@ get_and_clean_revenue_data <- function() { revenue_data <- data.frame() if (has_revenue) { - revenue_data <- read_rda(paste0(analysis_inputs_path, "/revenue_data_member_ticker.rda")) + revenue_data <- read_rda(file.path(analysis_inputs_path, "revenue_data_member_ticker.rda")) # col_types = "dcdcclcd") revenue_data <- revenue_data %>% @@ -1010,7 +1010,7 @@ get_and_clean_revenue_data <- function() { } get_and_clean_company_fin_data <- function() { - comp_fin_data_raw <- read_rds(paste0(analysis_inputs_path, "/consolidated_financial_data.rda")) + comp_fin_data_raw <- read_rds(file.path(analysis_inputs_path, "consolidated_financial_data.rda")) comp_fin_data_raw <- comp_fin_data_raw %>% select( company_id, company_name, bloomberg_id, country_of_domicile, corporate_bond_ticker, bics_subgroup, bics_sector, @@ -1026,7 +1026,7 @@ get_and_clean_company_fin_data <- function() { } get_and_clean_debt_fin_data <- function() { - debt_fin_data_raw <- read_rds(paste0(analysis_inputs_path, "/debt_financial_data.rda")) + debt_fin_data_raw <- read_rds(file.path(analysis_inputs_path, "debt_financial_data.rda")) return(debt_fin_data_raw) } @@ -1397,7 +1397,7 @@ create_audit_chart <- function(audit_file, proc_input_path) { ) - ggsave(paste0(proc_input_path, "/AuditChart.png"), Chart_1, height = nrow_Legend, width = 8) + ggsave(file.path(proc_input_path, "AuditChart.png"), Chart_1, height = nrow_Legend, width = 8) } create_audit_file <- function(portfolio_total) { @@ -1531,7 +1531,7 @@ get_average_emission_data <- function(inc_emission_factors) { average_sector_intensity <- data.frame() if (inc_emission_factors) { - average_sector_intensity <- read_rda(paste0(analysis_inputs_path, "/average_sector_intensity.rda")) + average_sector_intensity <- read_rda(file.path(analysis_inputs_path, "average_sector_intensity.rda")) } return(average_sector_intensity) } @@ -1540,7 +1540,7 @@ get_company_emission_data <- function(inc_emission_factors) { company_emissions <- data.frame() if (inc_emission_factors) { - company_emissions <- read_rda(paste0(analysis_inputs_path, "/company_emissions.rda")) + company_emissions <- read_rda(file.path(analysis_inputs_path, "company_emissions.rda")) } return(company_emissions) } From 81fad55eededa03498c08b99ebbf866cc34f6a58 Mon Sep 17 00:00:00 2001 From: Mauro Lepore Date: Thu, 18 Feb 2021 12:58:13 -0600 Subject: [PATCH 11/12] Revert "replace paste0 with file.path" This reverts commit 78688576ef34829d798cf3eaff12d67931a7da19. --- 0_portfolio_input_check_functions.R | 40 ++++++++++++++--------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/0_portfolio_input_check_functions.R b/0_portfolio_input_check_functions.R index 2d9ceece..4c822db8 100644 --- a/0_portfolio_input_check_functions.R +++ b/0_portfolio_input_check_functions.R @@ -4,29 +4,29 @@ read_raw_portfolio_file <- function(project_name) { portfolio <- NA - input_path <- file.path(project_location, "20_Raw_Inputs") + input_path <- paste0(project_location, "/20_Raw_Inputs/") csv_to_read <- list.files(path = input_path, pattern = "_Input.csv") txt_to_read <- list.files(path = input_path, pattern = "_Input.txt") if (length(csv_to_read) == 1) { - portfolio <- read_csv(file.path(input_path, csv_to_read)) + portfolio <- read_csv(paste0(input_path, csv_to_read)) } if (length(txt_to_read) == 1) { - enc <- guess_encoding(file.path(input_path, txt_to_read))$encoding[1] - portfolio <- read.table(file.path(input_path, txt_to_read), sep = ",", header = T, fileEncoding = enc) + enc <- guess_encoding(paste0(input_path, txt_to_read))$encoding[1] + portfolio <- read.table(paste0(input_path, txt_to_read), sep = ",", header = T, fileEncoding = enc) } # Reads in Files saved with a ; not a , if (ncol(portfolio) == 1 & length(csv_to_read) == 1) { - portfolio <- read.csv(file.path(input_path, csv_to_read), strip.white = T, stringsAsFactors = F, sep = ";") + portfolio <- read.csv(paste0(input_path, csv_to_read), strip.white = T, stringsAsFactors = F, sep = ";") } if (ncol(portfolio) == 1 & length(txt_to_read) == 1) { - portfolio <- read.table(file.path(input_path, txt_to_read), sep = "\t", header = T, fileEncoding = enc) + portfolio <- read.table(paste0(input_path, txt_to_read), sep = "\t", header = T, fileEncoding = enc) } if (ncol(portfolio) == 1 & length(txt_to_read) == 1) { - portfolio <- read.table(file.path(input_path, txt_to_read), sep = ";", header = T, fileEncoding = enc) + portfolio <- read.table(paste0(input_path, txt_to_read), sep = ";", header = T, fileEncoding = enc) } @@ -882,13 +882,13 @@ get_and_clean_currency_data <- function() { get_and_clean_fund_data <- function() { fund_data <- NA # Fund Data - if (file.exists(file.path(analysis_inputs_path, "fund_data_", financial_timestamp, ".rda"))) { - fund_data <- readRDS(file.path(analysis_inputs_path, "fund_data_", financial_timestamp, ".rda")) - } else if (file.exists(file.path(analysis_inputs_path, "fund_data_2018Q4.rda"))) { - fund_data <- readRDS(file.path(analysis_inputs_path, "fund_data_2018Q4.rda")) + if (file.exists(paste0(analysis_inputs_path, "/fund_data_", financial_timestamp, ".rda"))) { + fund_data <- readRDS(paste0(analysis_inputs_path, "/fund_data_", financial_timestamp, ".rda")) + } else if (file.exists(paste0(analysis_inputs_path, "/fund_data_2018Q4.rda"))) { + fund_data <- readRDS(paste0(analysis_inputs_path, "/fund_data_2018Q4.rda")) print("Old Fund Data being used. Replace FundsData2018Q4 or check name of file.") - } else if (file.exists(file.path(analysis_inputs_path, "SFC_26052020_funds.csv"))) { - fund_data <- read_csv(file.path(analysis_inputs_path, "SFC_26052020_funds.csv")) + } else if (file.exists(paste0(analysis_inputs_path, "/SFC_26052020_funds.csv"))) { + fund_data <- read_csv(paste0(analysis_inputs_path, "/SFC_26052020_funds.csv")) print("2020Q2 SFC fund data being used") } else { if (!data_check(fund_data)) { @@ -909,7 +909,7 @@ get_and_clean_fund_data <- function() { get_and_clean_fin_data <- function(fund_data) { # Financial Data - fin_data_raw <- read_rda(file.path(analysis_inputs_path, "security_financial_data.rda")) %>% as_tibble() + fin_data_raw <- read_rda(paste0(analysis_inputs_path, "/security_financial_data.rda")) %>% as_tibble() # remove unclear duplicates from raw financial data. This should be moved to DataStore. rm_duplicates <- read_csv("non_distinct_isins.csv") @@ -997,7 +997,7 @@ get_and_clean_revenue_data <- function() { revenue_data <- data.frame() if (has_revenue) { - revenue_data <- read_rda(file.path(analysis_inputs_path, "revenue_data_member_ticker.rda")) + revenue_data <- read_rda(paste0(analysis_inputs_path, "/revenue_data_member_ticker.rda")) # col_types = "dcdcclcd") revenue_data <- revenue_data %>% @@ -1010,7 +1010,7 @@ get_and_clean_revenue_data <- function() { } get_and_clean_company_fin_data <- function() { - comp_fin_data_raw <- read_rds(file.path(analysis_inputs_path, "consolidated_financial_data.rda")) + comp_fin_data_raw <- read_rds(paste0(analysis_inputs_path, "/consolidated_financial_data.rda")) comp_fin_data_raw <- comp_fin_data_raw %>% select( company_id, company_name, bloomberg_id, country_of_domicile, corporate_bond_ticker, bics_subgroup, bics_sector, @@ -1026,7 +1026,7 @@ get_and_clean_company_fin_data <- function() { } get_and_clean_debt_fin_data <- function() { - debt_fin_data_raw <- read_rds(file.path(analysis_inputs_path, "debt_financial_data.rda")) + debt_fin_data_raw <- read_rds(paste0(analysis_inputs_path, "/debt_financial_data.rda")) return(debt_fin_data_raw) } @@ -1397,7 +1397,7 @@ create_audit_chart <- function(audit_file, proc_input_path) { ) - ggsave(file.path(proc_input_path, "AuditChart.png"), Chart_1, height = nrow_Legend, width = 8) + ggsave(paste0(proc_input_path, "/AuditChart.png"), Chart_1, height = nrow_Legend, width = 8) } create_audit_file <- function(portfolio_total) { @@ -1531,7 +1531,7 @@ get_average_emission_data <- function(inc_emission_factors) { average_sector_intensity <- data.frame() if (inc_emission_factors) { - average_sector_intensity <- read_rda(file.path(analysis_inputs_path, "average_sector_intensity.rda")) + average_sector_intensity <- read_rda(paste0(analysis_inputs_path, "/average_sector_intensity.rda")) } return(average_sector_intensity) } @@ -1540,7 +1540,7 @@ get_company_emission_data <- function(inc_emission_factors) { company_emissions <- data.frame() if (inc_emission_factors) { - company_emissions <- read_rda(file.path(analysis_inputs_path, "company_emissions.rda")) + company_emissions <- read_rda(paste0(analysis_inputs_path, "/company_emissions.rda")) } return(company_emissions) } From 4cde777bd459151942c9bf108f38eb96e9dc285f Mon Sep 17 00:00:00 2001 From: Mauro Lepore Date: Thu, 18 Feb 2021 19:28:25 +0000 Subject: [PATCH 12/12] Revert to file.path() --- 0_portfolio_input_check_functions.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/0_portfolio_input_check_functions.R b/0_portfolio_input_check_functions.R index 4c822db8..03ce65a1 100644 --- a/0_portfolio_input_check_functions.R +++ b/0_portfolio_input_check_functions.R @@ -1531,7 +1531,7 @@ get_average_emission_data <- function(inc_emission_factors) { average_sector_intensity <- data.frame() if (inc_emission_factors) { - average_sector_intensity <- read_rda(paste0(analysis_inputs_path, "/average_sector_intensity.rda")) + average_sector_intensity <- read_rda(file.path(analysis_inputs_path, "average_sector_intensity.rda")) } return(average_sector_intensity) } @@ -1540,7 +1540,7 @@ get_company_emission_data <- function(inc_emission_factors) { company_emissions <- data.frame() if (inc_emission_factors) { - company_emissions <- read_rda(paste0(analysis_inputs_path, "/company_emissions.rda")) + company_emissions <- read_rda(file.path(analysis_inputs_path, "company_emissions.rda")) } return(company_emissions) }