From 331a6aac7908ea07cab064ae817b45ee64640241 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 10 Dec 2024 14:55:47 +0100 Subject: [PATCH 01/33] Increment version number to 0.0.0.9007 --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index adb6b89..31b5b7c 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: workflow.pacta.dashboard Title: Run PACTA dashboard JSON generation -Version: 0.0.0.9006 +Version: 0.0.0.9007 Authors@R: c(person(given = "Alex", family = "Axthelm", From 91f8504ab5a42e5ec80a108b48348fa6db2c241d Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 11 Dec 2024 12:08:06 +0100 Subject: [PATCH 02/33] Enable param parsing in script --- DESCRIPTION | 9 +- NAMESPACE | 5 + R/prepare_pacta_dashboard_data.R | 2 + R/workflow.pacta.dashboard-package.R | 5 + docker-compose.yml | 9 +- inst/extdata/parameters/GENERAL_2022Q4.json | 6 + inst/extdata/parameters/GENERAL_2023Q4.json | 6 + inst/extdata/parameters/report_GENERAL.json | 7 + .../parameters/report_GENERAL_2022Q4.json | 8 + .../parameters/report_GENERAL_2023Q4.json | 8 + inst/extdata/parameters/report_default.json | 42 ++++ inst/extdata/schema/portfolio.json | 36 ++++ inst/extdata/schema/rawParameters.json | 31 +++ inst/extdata/schema/reportingParameters.json | 180 ++++++++++++++++++ inst/extdata/scripts/prepare_dashboard_data.R | 58 +++--- 15 files changed, 379 insertions(+), 33 deletions(-) create mode 100644 inst/extdata/parameters/GENERAL_2022Q4.json create mode 100644 inst/extdata/parameters/GENERAL_2023Q4.json create mode 100644 inst/extdata/parameters/report_GENERAL.json create mode 100644 inst/extdata/parameters/report_GENERAL_2022Q4.json create mode 100644 inst/extdata/parameters/report_GENERAL_2023Q4.json create mode 100644 inst/extdata/parameters/report_default.json create mode 100644 inst/extdata/schema/portfolio.json create mode 100644 inst/extdata/schema/rawParameters.json create mode 100644 inst/extdata/schema/reportingParameters.json diff --git a/DESCRIPTION b/DESCRIPTION index 31b5b7c..6478d8f 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -29,11 +29,16 @@ Imports: dplyr, glue, jsonlite, + logger, magrittr, pacta.portfolio.utils, + pacta.workflow.utils, readr, rlang, stringr, - tidyr + tidyr, + workflow.pacta Remotes: - RMI-PACTA/pacta.portfolio.utils + RMI-PACTA/pacta.portfolio.utils, + RMI-PACTA/pacta.workflow.utils, + RMI-PACTA/workflow.pacta diff --git a/NAMESPACE b/NAMESPACE index 7334320..7eff832 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -30,6 +30,11 @@ importFrom(dplyr,summarise) importFrom(dplyr,tibble) importFrom(dplyr,transmute) importFrom(dplyr,ungroup) +importFrom(logger,log_debug) +importFrom(logger,log_error) +importFrom(logger,log_info) +importFrom(logger,log_trace) +importFrom(logger,log_warn) importFrom(magrittr,"%>%") importFrom(rlang,":=") importFrom(rlang,.data) diff --git a/R/prepare_pacta_dashboard_data.R b/R/prepare_pacta_dashboard_data.R index bfb82f9..0b5f8a6 100644 --- a/R/prepare_pacta_dashboard_data.R +++ b/R/prepare_pacta_dashboard_data.R @@ -5,6 +5,8 @@ prepare_pacta_dashboard_data <- function( benchmarks_dir = Sys.getenv("BENCHMARKS_DIR") ) { +log_info("Preparing data for the PACTA dashboard.") + # input and output directories ------------------------------------------------- input_dir <- analysis_output_dir diff --git a/R/workflow.pacta.dashboard-package.R b/R/workflow.pacta.dashboard-package.R index ac090cb..4fd1011 100644 --- a/R/workflow.pacta.dashboard-package.R +++ b/R/workflow.pacta.dashboard-package.R @@ -32,6 +32,11 @@ #' @importFrom dplyr tibble #' @importFrom dplyr transmute #' @importFrom dplyr ungroup +#' @importFrom logger log_debug +#' @importFrom logger log_error +#' @importFrom logger log_info +#' @importFrom logger log_trace +#' @importFrom logger log_warn #' @importFrom magrittr %>% #' @importFrom rlang := #' @importFrom rlang .data diff --git a/docker-compose.yml b/docker-compose.yml index 50a095e..8960d0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,11 +2,10 @@ services: workflow.pacta.dashboard: build: . - # stdin_open: true - # tty: true - # entrypoint: ["bash"] - # entrypoint: ["R", "--args"] - # command: '{\"portfolio\": {\"files\": [\"default_portfolio.csv\"], \"holdingsDate\": \"2023-12-31\", \"name\": \"FooPortfolio\"}, \"inherit\": \"GENERAL_2023Q4\"}' + stdin_open: true + tty: true + entrypoint: ["R", "--args"] + command: '{\"portfolio\": {\"files\": [\"default_portfolio.csv\"], \"holdingsDate\": \"2023-12-31\", \"name\": \"FooPortfolio\"}, \"inherit\": \"GENERAL_2023Q4\"}' environment: LOG_LEVEL: TRACE ANALYSIS_OUTPUT_DIR: "/mnt/analysis_output_dir" diff --git a/inst/extdata/parameters/GENERAL_2022Q4.json b/inst/extdata/parameters/GENERAL_2022Q4.json new file mode 100644 index 0000000..e187151 --- /dev/null +++ b/inst/extdata/parameters/GENERAL_2022Q4.json @@ -0,0 +1,6 @@ +{ + "inherit": [ + "analysis_GENERAL_2022Q4", + "report_GENERAL_2022Q4" + ] +} diff --git a/inst/extdata/parameters/GENERAL_2023Q4.json b/inst/extdata/parameters/GENERAL_2023Q4.json new file mode 100644 index 0000000..11ee25f --- /dev/null +++ b/inst/extdata/parameters/GENERAL_2023Q4.json @@ -0,0 +1,6 @@ +{ + "inherit": [ + "analysis_GENERAL_2023Q4", + "report_GENERAL_2023Q4" + ] +} diff --git a/inst/extdata/parameters/report_GENERAL.json b/inst/extdata/parameters/report_GENERAL.json new file mode 100644 index 0000000..7bf665c --- /dev/null +++ b/inst/extdata/parameters/report_GENERAL.json @@ -0,0 +1,7 @@ +{ + "reporting": { + "projectCode": "GENERAL", + "projectReportName": "general" + }, + "inherit": "report_default" +} diff --git a/inst/extdata/parameters/report_GENERAL_2022Q4.json b/inst/extdata/parameters/report_GENERAL_2022Q4.json new file mode 100644 index 0000000..d7ac08d --- /dev/null +++ b/inst/extdata/parameters/report_GENERAL_2022Q4.json @@ -0,0 +1,8 @@ +{ + "reporting": { + "scenarioOther": "GECO2022_1.5C", + "selectScenario": "WEO2022_NZE_2050" + }, + "inherit": "report_GENERAL" +} + diff --git a/inst/extdata/parameters/report_GENERAL_2023Q4.json b/inst/extdata/parameters/report_GENERAL_2023Q4.json new file mode 100644 index 0000000..d7ac08d --- /dev/null +++ b/inst/extdata/parameters/report_GENERAL_2023Q4.json @@ -0,0 +1,8 @@ +{ + "reporting": { + "scenarioOther": "GECO2022_1.5C", + "selectScenario": "WEO2022_NZE_2050" + }, + "inherit": "report_GENERAL" +} + diff --git a/inst/extdata/parameters/report_default.json b/inst/extdata/parameters/report_default.json new file mode 100644 index 0000000..a7a6f57 --- /dev/null +++ b/inst/extdata/parameters/report_default.json @@ -0,0 +1,42 @@ +{ + "reporting": { + "scenarioGeography": "Global", + "sectorList": [ + "Power", + "Automotive", + "Oil&Gas", + "Coal", + "Steel", + "Aviation", + "Cement" + ], + "portfolioAllocationMethod": "portfolio_weight", + "techRoadmapSectors": [ + "Power", + "Automotive", + "Oil&Gas", + "Coal" + ], + "pactaSectorsNotAnalysed": [ + "Steel", + "Aviation", + "Cement" + ], + "greenTechs": [ + "RenewablesCap", + "HydroCap", + "NuclearCap", + "Hybrid", + "Electric", + "FuelCell", + "Electric Arc Furnace" + ] + }, + "user": { + "currencyExchangeValue": 1, + "displayCurrency": "USD", + "name": "Default Investor", + "languageSelect": "EN", + "peerGroup": "bank" + } +} diff --git a/inst/extdata/schema/portfolio.json b/inst/extdata/schema/portfolio.json new file mode 100644 index 0000000..cc70aaa --- /dev/null +++ b/inst/extdata/schema/portfolio.json @@ -0,0 +1,36 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/RMI-PACTA/workflow.pacta/main/inst/extdata/schema/portfolio.json", + "title": "portfolio", + "description": "Definition for PACTA portfolio object", + "$comment": "Created by Alex Axthelm, aaxthelm@rmi.org", + "type": "object", + "properties": { + "files": { + "type": "array", + "description": "List of portfolio files to be analyzed.", + "items": { + "type": "string", + "description": "Path to the file." + } + }, + "holdingsDate": { + "type": "string", + "description": "Date of the holdings data.", + "format": "date", + "enum": [ + "2022-12-31", + "2023-12-31" + ] + }, + "name": { + "type": "string", + "description": "Name of the portfolio." + } + }, + "required": [ + "files", + "holdingsDate" + ], + "additionalProperties": false +} diff --git a/inst/extdata/schema/rawParameters.json b/inst/extdata/schema/rawParameters.json new file mode 100644 index 0000000..1515ed1 --- /dev/null +++ b/inst/extdata/schema/rawParameters.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/RMI-PACTA/workflow.pacta/main/inst/extdata/schema/rawParameters.json", + "title": "rawPortfolioParameters", + "description": "Valid input parameters for the PACTA workflow. Less than full parameters, since PACTA handles inheritence.", + "$comment": "Created by Alex Axthelm, aaxthelm@rmi.org", + "type": "object", + "properties": { + "portfolio": { + "$ref": "portfolio.json" + }, + "inherit": { + "type": "string", + "description": "Path to a file with inheritance information." + } + }, + "anyOf": [ + { + "required": [ + "inherit" + ] + }, + { + "$ref": "reportingParameters.json" + } + ], + "required": [ + "portfolio" + ], + "additionalProperties": true +} diff --git a/inst/extdata/schema/reportingParameters.json b/inst/extdata/schema/reportingParameters.json new file mode 100644 index 0000000..28962b3 --- /dev/null +++ b/inst/extdata/schema/reportingParameters.json @@ -0,0 +1,180 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/RMI-PACTA/workflow.pacta/main/inst/extdata/schema/portfolioParameters.json", + "title": "portfolioParameters", + "description": "Full Parameters required to run workflow.pacta.", + "$comment": "Created by Alex Axthelm, aaxthelm@rmi.org", + "type": "object", + "properties": { + "portfolio": { + "$ref": "portfolio.json" + }, + "user": { + "type": "object", + "description": "User information.", + "properties": { + "name": { + "type": "string", + "description": "User name." + }, + "languageSelect": { + "type": "string", + "description": "Language selection.", + "enum": [ + "EN", + "ES", + "FR", + "DE" + ] + }, + "peerGroup": { + "type": "string", + "description": "Peer group selection.", + "enum": [ + "assetmanager", + "bank", + "insurance", + "other", + "pensionfund" + ] + }, + "currencyExchangeValue": { + "type": "number", + "description": "Currency exchange value for displayCurrency", + "exclusiveMinimum": 0, + "default": 1 + }, + "displayCurrency": { + "type": "string", + "description": "Currency for display. ISO 4217 currency code.", + "pattern": "^[A-Z]{3}$" + } + }, + "required": [ + "name", + "languageSelect", + "peerGroup", + "currencyExchangeValue", + "displayCurrency" + ], + "additionalProperties": true + } + }, + "reporting": { + "type": "object", + "description": "Parameters for reporting.", + "properties": { + "projectCode": { + "type": "string", + "description": "Project code.", + "enum": [ + "GENERAL" + ] + }, + "projectReportName": { + "type": "string", + "description": "Project code.", + "enum": [ + "general" + ] + }, + "greenTechs": { + "type": "array", + "description": "Green technologies to be included in the report.", + "items": { + "type": "string", + "enum": [ + "RenewablesCap", + "HydroCap", + "NuclearCap", + "Hybrid", + "Electric", + "FuelCell", + "Electric Arc Furnace" + ] + } + }, + "pactaSectorsNotAnalysed": { + "type": "array", + "description": "Sectors not analysed in the report.", + "items": { + "type": "string", + "enum": [ + "Automotive", + "Aviation", + "Cement", + "Coal", + "Oil&Gas", + "Power", + "Steel" + ] + } + }, + "portfolioAllocationMethod": { + "type": "string", + "description": "Portfolio allocation method.", + "enum": [ + "portfolio_weight", + "ownership_weight" + ] + }, + "scenarioGeography": { + "type": "string", + "description": "Scenario geography." + }, + "scenarioOther": { + "type": "string", + "description": "Scenario other." + }, + "selectScenario": { + "type": "string", + "description": "default scenario to display." + }, + "techRoadmapSectors": { + "type": "array", + "description": "Sectors with technology roadmap to be included in the report.", + "items": { + "type": "string", + "enum": [ + "Automotive", + "Aviation", + "Cement", + "Coal", + "Oil&Gas", + "Power", + "Steel" + ] + } + } + }, + "required": [ + "projectCode", + "projectReportName", + "greenTechs", + "pactaSectorsNotAnalysed", + "portfolioAllocationMethod", + "scenarioGeography", + "scenarioOther", + "selectScenario", + "techRoadmapSectors" + ], + "additionalProperties": true + }, + "allOf": [ + { + "required": [ + "portfolio", + "reporting", + "user" + ] + }, + { + "not": { + "required": [ + "inherit" + ] + } + } + ], + "additionalProperties": true +} diff --git a/inst/extdata/scripts/prepare_dashboard_data.R b/inst/extdata/scripts/prepare_dashboard_data.R index 7273865..bd4fdd5 100644 --- a/inst/extdata/scripts/prepare_dashboard_data.R +++ b/inst/extdata/scripts/prepare_dashboard_data.R @@ -1,31 +1,37 @@ -# logger::log_threshold(Sys.getenv("LOG_LEVEL", "INFO")) +logger::log_threshold(Sys.getenv("LOG_LEVEL", "INFO")) -# raw_params <- commandArgs(trailingOnly = TRUE) -# params <- pacta.workflow.utils::parse_raw_params( -# json = raw_params, -# inheritence_search_paths = system.file( -# "extdata", "parameters", -# package = "workflow.pacta.report" -# ) , -# schema_file = system.file( -# "extdata", "schema", "reportingParameters.json", -# package = "workflow.pacta.report" -# ), -# raw_schema_file = system.file( -# "extdata", "schema", "rawParameters.json", -# package = "workflow.pacta.report" -# ), -# force_array = c("portfolio", "files") -# ) +raw_params <- commandArgs(trailingOnly = TRUE) +params <- pacta.workflow.utils::parse_raw_params( + json = raw_params, + inheritence_search_paths = c( + system.file( + "extdata", "parameters", + package = "workflow.pacta.dashboard" + ), + system.file( + "extdata", "parameters", + package = "workflow.pacta" + ) + ), + schema_file = system.file( + "extdata", "schema", "reportingParameters.json", + package = "workflow.pacta.dashboard" + ), + raw_schema_file = system.file( + "extdata", "schema", "rawParameters.json", + package = "workflow.pacta.dashboard" + ), + force_array = c("portfolio", "files") +) manifest_info <- workflow.pacta.dashboard:::build_dashboard( - # params = params + params = params ) -# pacta.workflow.utils::export_manifest( -# input_files = manifest_info[["input_files"]], -# output_files = manifest_info[["output_files"]], -# params = manifest_info[["params"]], -# manifest_path = file.path(Sys.getenv("REPORT_OUTPUT_DIR"), "manifest.json"), -# raw_params = raw_params -# ) +pacta.workflow.utils::export_manifest( + input_files = manifest_info[["input_files"]], + output_files = manifest_info[["output_files"]], + params = manifest_info[["params"]], + manifest_path = file.path(Sys.getenv("REPORT_OUTPUT_DIR"), "manifest.json"), + raw_params = raw_params +) From 57ca87f0daccf69863e3d10f1be2a781b509bdb1 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 11 Dec 2024 12:14:06 +0100 Subject: [PATCH 03/33] Use arguments for directories, rather than reassign --- R/prepare_pacta_dashboard_data.R | 63 ++++++++++++++------------------ 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/R/prepare_pacta_dashboard_data.R b/R/prepare_pacta_dashboard_data.R index 0b5f8a6..8155d81 100644 --- a/R/prepare_pacta_dashboard_data.R +++ b/R/prepare_pacta_dashboard_data.R @@ -7,13 +7,6 @@ prepare_pacta_dashboard_data <- function( log_info("Preparing data for the PACTA dashboard.") -# input and output directories ------------------------------------------------- - -input_dir <- analysis_output_dir -output_dir <- dashboard_data_dir -data_dir <- benchmarks_dir - - # portfolio/user parameters ---------------------------------------------------- investor_name <- "investor_name" @@ -43,7 +36,7 @@ all_tech_levels = c(power_tech_levels, auto_levels, oil_gas_levels, coal_levels, # config parameters from manifest ---------------------------------------------- -manifest <- jsonlite::read_json(path = file.path(input_dir, "manifest.json")) +manifest <- jsonlite::read_json(path = file.path(analysis_output_dir, "manifest.json")) start_year <- manifest$params$analysis$startYear year_span <- manifest$params$analysis$timeHorizon @@ -54,18 +47,18 @@ scen_geo_levels <- unlist(manifest$params$analysis$scenarioGeographiesList) # load results from input directory -------------------------------------------- -audit_file <- readRDS(file.path(input_dir, "audit_file.rds")) -emissions <- readRDS(file.path(input_dir, "emissions.rds")) -equity_results_portfolio <- readRDS(file.path(input_dir, "Equity_results_portfolio.rds")) -bonds_results_portfolio <- readRDS(file.path(input_dir, "Bonds_results_portfolio.rds")) -equity_results_company <- readRDS(file.path(input_dir, "Equity_results_company.rds")) -bonds_results_company <- readRDS(file.path(input_dir, "Bonds_results_company.rds")) +audit_file <- readRDS(file.path(analysis_output_dir, "audit_file.rds")) +emissions <- readRDS(file.path(analysis_output_dir, "emissions.rds")) +equity_results_portfolio <- readRDS(file.path(analysis_output_dir, "Equity_results_portfolio.rds")) +bonds_results_portfolio <- readRDS(file.path(analysis_output_dir, "Bonds_results_portfolio.rds")) +equity_results_company <- readRDS(file.path(analysis_output_dir, "Equity_results_company.rds")) +bonds_results_company <- readRDS(file.path(analysis_output_dir, "Bonds_results_company.rds")) # data from PACTA inputs used to generate the results -------------------------- -indices_bonds_results_portfolio <- readRDS(file.path(data_dir, "Indices_bonds_results_portfolio.rds")) -indices_equity_results_portfolio <- readRDS(file.path(data_dir, "Indices_equity_results_portfolio.rds")) +indices_bonds_results_portfolio <- readRDS(file.path(benchmarks_dir, "Indices_bonds_results_portfolio.rds")) +indices_equity_results_portfolio <- readRDS(file.path(benchmarks_dir, "Indices_equity_results_portfolio.rds")) peers_bonds_results_portfolio <- pacta.portfolio.utils::empty_portfolio_results() peers_equity_results_portfolio <- pacta.portfolio.utils::empty_portfolio_results() @@ -156,7 +149,7 @@ audit_file %>% ) %>% translate_df_contents("data_included_table", dictionary, inplace = TRUE) %>% translate_df_headers("data_included_table", language_select, header_dictionary) %>% - jsonlite::write_json(path = file.path(output_dir, "data_included_table.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_included_table.json")) # data_value_pie_bonds.json ---------------------------------------------------- @@ -170,7 +163,7 @@ audit_file %>% currency_exchange_value = currency_exchange_value ) %>% translate_df_contents("data_value_pie_bonds", dictionary) %>% - jsonlite::write_json(path = file.path(output_dir, "data_value_pie_bonds.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_value_pie_bonds.json")) # data_emissions_equity.json --------------------------------------------------- @@ -183,7 +176,7 @@ emissions %>% pacta_sectors = pacta_sectors ) %>% translate_df_contents("data_emissions_pie_equity", dictionary) %>% - jsonlite::write_json(path = file.path(output_dir, "data_emissions_pie_equity.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_emissions_pie_equity.json")) # data_emissions_bonds.json ---------------------------------------------------- @@ -196,7 +189,7 @@ emissions %>% pacta_sectors = pacta_sectors ) %>% translate_df_contents("data_emissions_pie_bonds", dictionary) %>% - jsonlite::write_json(path = file.path(output_dir, "data_emissions_pie_bonds.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_emissions_pie_bonds.json")) # data_value_pie_equity.json --------------------------------------------------- @@ -210,7 +203,7 @@ audit_file %>% currency_exchange_value = currency_exchange_value ) %>% translate_df_contents("data_value_pie_equity", dictionary) %>% - jsonlite::write_json(path = file.path(output_dir, "data_value_pie_equity.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_value_pie_equity.json")) # data_techmix.json ------------------------------------------------------------ @@ -233,7 +226,7 @@ prep_techexposure( all_tech_levels = all_tech_levels ) %>% translate_df_contents("techexposure_data", dictionary) %>% - jsonlite::write_json(path = file.path(output_dir, "data_techexposure.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_techexposure.json")) # data_techmix_sector.json ----------------------------------------------------- @@ -253,7 +246,7 @@ prep_techmix_sector( green_techs, all_tech_levels ) %>% - jsonlite::write_json(path = file.path(output_dir, "data_techmix_sector.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_techmix_sector.json")) # data_trajectory_alignment.json ----------------------------------------------- @@ -274,7 +267,7 @@ prep_trajectory_alignment( all_tech_levels = all_tech_levels ) %>% translate_df_contents("data_trajectory_alignment", dictionary) %>% - jsonlite::write_json(path = file.path(output_dir, "data_trajectory_alignment.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_trajectory_alignment.json")) # data_emissions.json ---------------------------------------------------------- @@ -288,7 +281,7 @@ prep_emissions_trajectory( start_year = start_year ) %>% translate_df_contents("data_emissions", dictionary) %>% - jsonlite::write_json(path = file.path(output_dir, "data_emissions.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_emissions.json")) # data_exposure_stats.json @@ -299,7 +292,7 @@ prep_exposure_stats( pacta_sectors = pacta_sectors, currency_exchange_value = currency_exchange_value ) %>% - jsonlite::write_json(path = file.path(output_dir, "data_exposure_stats.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_exposure_stats.json")) # data_company_bubble.json ----------------------------------------------------- @@ -312,7 +305,7 @@ prep_company_bubble( green_techs = green_techs ) %>% translate_df_contents("data_company_bubble", dictionary) %>% - jsonlite::write_json(path = file.path(output_dir, "data_company_bubble.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_company_bubble.json")) # data_techexposure_company_companies.json ------------------------------------- @@ -326,7 +319,7 @@ prep_key_bars_company( all_tech_levels = all_tech_levels ) %>% translate_df_contents("data_key_bars_company", dictionary) %>% - jsonlite::write_json(path = file.path(output_dir, "data_techexposure_company_companies.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_techexposure_company_companies.json")) # data_techexposure_company_portfolio.json ------------------------------------- @@ -340,32 +333,32 @@ prep_key_bars_portfolio( all_tech_levels = all_tech_levels ) %>% translate_df_contents("data_key_bars_portfolio", dictionary) %>% - jsonlite::write_json(path = file.path(output_dir, "data_techexposure_company_portfolio.json")) + jsonlite::write_json(path = file.path(dashboard_data_dir, "data_techexposure_company_portfolio.json")) # put JSON and CSV outputs into a zip archive ---------------------------------- -zip_outputs(output_dir) +zip_outputs(dashboard_data_dir) } -zip_outputs <- function(output_dir) { - json_filenames <- list.files(output_dir, pattern = "[.]json$") +zip_outputs <- function(dashboard_data_dir) { + json_filenames <- list.files(dashboard_data_dir, pattern = "[.]json$") zip_temp <- file.path(tempdir(), "zip_temp") dir.create(zip_temp, showWarnings = FALSE) for (json_filename in json_filenames) { file.copy( - from = file.path(output_dir, json_filename), + from = file.path(dashboard_data_dir, json_filename), to = file.path(zip_temp, json_filename) ) csv_filename <- sub("[.]json$", ".csv", json_filename) df <- jsonlite::read_json( - path = file.path(output_dir, json_filename), + path = file.path(dashboard_data_dir, json_filename), simplifyVector = TRUE ) @@ -380,7 +373,7 @@ zip_outputs <- function(output_dir) { } utils::zip( - zipfile = file.path(output_dir, "archive.zip"), + zipfile = file.path(dashboard_data_dir, "archive.zip"), files = list.files(zip_temp, full.names = TRUE), flags = "-r9Xjq" ) From 8a309ee28bf0a5de2a9671e612b147b5226b56b4 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 11 Dec 2024 13:14:53 +0100 Subject: [PATCH 04/33] Use params for report params, and add simple logging --- R/build_dashboard.R | 3 ++ R/prepare_pacta_dashboard_data.R | 47 ++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/R/build_dashboard.R b/R/build_dashboard.R index 9121a83..4a511f7 100644 --- a/R/build_dashboard.R +++ b/R/build_dashboard.R @@ -8,11 +8,13 @@ build_dashboard <- function( ) { prepare_pacta_dashboard_data( + params = params, analysis_output_dir = analysis_output_dir, dashboard_data_dir = dashboard_data_dir, benchmarks_dir = benchmarks_dir ) + log_info("Copying dashboard files.") dashboard_copy_success <- copy_dashboard_files( dashboard_skeleton_files_dir = dashboard_skeleton_files_dir, dashboard_output_dir = dashboard_output_dir @@ -20,6 +22,7 @@ build_dashboard <- function( dashboard_output_data_dir <- file.path(dashboard_output_dir, "data") if (dashboard_data_dir != dashboard_output_data_dir) { + log_info("Copying dashboard data to correct location.") if (!dir.exists(dashboard_output_data_dir)) { dir.create(dashboard_output_data_dir, recursive = TRUE) } diff --git a/R/prepare_pacta_dashboard_data.R b/R/prepare_pacta_dashboard_data.R index 8155d81..b4ff84e 100644 --- a/R/prepare_pacta_dashboard_data.R +++ b/R/prepare_pacta_dashboard_data.R @@ -8,21 +8,22 @@ prepare_pacta_dashboard_data <- function( log_info("Preparing data for the PACTA dashboard.") # portfolio/user parameters ---------------------------------------------------- +log_debug("Reading portfolio/user parameters.") -investor_name <- "investor_name" -portfolio_name <- "portfolio_name" -peer_group <- "peer_group" -language_select <- "EN" +investor_name <- params[["user"]][["investor_name"]] +portfolio_name <- params[["portfolio"]][["name"]] +peer_group <- params[["user"]][["peerGroup"]] +language_select <- params[["user"]][["languageSelect"]] -currency_exchange_value <- 1 -display_currency <- "USD" +currency_exchange_value <- params[["user"]][["currencyExchangeValue"]] +display_currency <- params[["user"]][["displayCurrency"]] -select_scenario_other <- "WEO2023_NZE_2050" -select_scenario <- "WEO2023_NZE_2050" +select_scenario_other <- params[["reporting"]][["selectOther"]] +select_scenario <- params[["reporting"]][["selectScenario"]] -green_techs <- c("RenewablesCap", "HydroCap", "NuclearCap", "Hybrid", "Electric", "FuelCell", "Hybrid_HDV", "Electric_HDV", "FuelCell_HDV","Electric Arc Furnace") -tech_roadmap_sectors <- c("Automotive", "Power", "Oil&Gas", "Coal") -pacta_sectors_not_analysed <- c("Steel", "Aviation", "Cement") +green_techs <- params[["reporting"]][["greenTechs"]] +tech_roadmap_sectors <- params[["reporting"]][["techRoadmapSectors"]] +pacta_sectors_not_analysed <- params[["reporting"]][["pactaSectorsNotAnalysed"]] power_tech_levels = c("RenewablesCap", "HydroCap", "NuclearCap", "GasCap", "OilCap", "CoalCap") oil_gas_levels = c("Oil", "Gas") @@ -35,6 +36,7 @@ all_tech_levels = c(power_tech_levels, auto_levels, oil_gas_levels, coal_levels, # config parameters from manifest ---------------------------------------------- +log_debug("Reading config parameters from analysis outputs manifest.") manifest <- jsonlite::read_json(path = file.path(analysis_output_dir, "manifest.json")) @@ -46,6 +48,7 @@ scen_geo_levels <- unlist(manifest$params$analysis$scenarioGeographiesList) # load results from input directory -------------------------------------------- +log_debug("Loading results from input directory.") audit_file <- readRDS(file.path(analysis_output_dir, "audit_file.rds")) emissions <- readRDS(file.path(analysis_output_dir, "emissions.rds")) @@ -56,14 +59,18 @@ bonds_results_company <- readRDS(file.path(analysis_output_dir, "Bonds_results_c # data from PACTA inputs used to generate the results -------------------------- +log_debug("Loading benchmark results.") indices_bonds_results_portfolio <- readRDS(file.path(benchmarks_dir, "Indices_bonds_results_portfolio.rds")) indices_equity_results_portfolio <- readRDS(file.path(benchmarks_dir, "Indices_equity_results_portfolio.rds")) + +log_debug("Loading peer results.") peers_bonds_results_portfolio <- pacta.portfolio.utils::empty_portfolio_results() peers_equity_results_portfolio <- pacta.portfolio.utils::empty_portfolio_results() # translations ----------------------------------------------------------------- +log_debug("Loading translations.") dataframe_translations <- readr::read_csv( system.file("extdata/translation/dataframe_labels.csv", package = "workflow.pacta.dashboard"), @@ -95,6 +102,7 @@ header_dictionary <- replace_contents(header_dictionary, display_currency) # add investor_name and portfolio_name to results data frames because ---------- # pacta.portfolio.report functions expect that --------------------------------- +log_debug("Adding investor_name and portfolio_name to results data frames.") audit_file <- audit_file %>% @@ -141,6 +149,7 @@ bonds_results_company <- # data_included_table.json ----------------------------------------------------- +log_info("Preparing data_included_table.json.") audit_file %>% prep_audit_table( investor_name = investor_name, @@ -154,6 +163,7 @@ audit_file %>% # data_value_pie_bonds.json ---------------------------------------------------- +log_info("Preparing data_value_pie_bonds.json.") audit_file %>% prep_exposure_pie( asset_type = "Bonds", @@ -168,6 +178,7 @@ audit_file %>% # data_emissions_equity.json --------------------------------------------------- +log_info("Preparing data_emissions_equity.json.") emissions %>% prep_emissions_pie( asset_type = "Equity", @@ -181,6 +192,7 @@ emissions %>% # data_emissions_bonds.json ---------------------------------------------------- +log_info("Preparing data_emissions_bonds.json.") emissions %>% prep_emissions_pie( asset_type = "Bonds", @@ -194,6 +206,7 @@ emissions %>% # data_value_pie_equity.json --------------------------------------------------- +log_info("Preparing data_value_pie_equity.json.") audit_file %>% prep_exposure_pie( asset_type = "Equity", @@ -208,6 +221,7 @@ audit_file %>% # data_techmix.json ------------------------------------------------------------ +log_info("Preparing data_techmix.json.") prep_techexposure( equity_results_portfolio = equity_results_portfolio, bonds_results_portfolio = bonds_results_portfolio, @@ -231,6 +245,7 @@ prep_techexposure( # data_techmix_sector.json ----------------------------------------------------- +log_info("Preparing data_techmix_sector.json.") prep_techmix_sector( equity_results_portfolio, bonds_results_portfolio, @@ -250,6 +265,7 @@ prep_techmix_sector( # data_trajectory_alignment.json ----------------------------------------------- +log_info("Preparing data_trajectory_alignment.json.") prep_trajectory_alignment( equity_results_portfolio = equity_results_portfolio, bonds_results_portfolio = bonds_results_portfolio, @@ -272,6 +288,7 @@ prep_trajectory_alignment( # data_emissions.json ---------------------------------------------------------- +log_info("Preparing data_emissions.json.") prep_emissions_trajectory( equity_results_portfolio = equity_results_portfolio, bonds_results_portfolio = bonds_results_portfolio, @@ -285,6 +302,7 @@ prep_emissions_trajectory( # data_exposure_stats.json +log_info("Preparing data_exposure_stats.json.") prep_exposure_stats( audit_file = audit_file, investor_name = investor_name, @@ -297,6 +315,7 @@ prep_exposure_stats( # data_company_bubble.json ----------------------------------------------------- +log_info("Preparing data_company_bubble.json.") prep_company_bubble( equity_results_company = equity_results_company, bonds_results_company = bonds_results_company, @@ -310,6 +329,7 @@ prep_company_bubble( # data_techexposure_company_companies.json ------------------------------------- +log_info("Preparing data_techexposure_company_companies.json.") prep_key_bars_company( equity_results_company = equity_results_company, bonds_results_company = bonds_results_company, @@ -324,6 +344,7 @@ prep_key_bars_company( # data_techexposure_company_portfolio.json ------------------------------------- +log_info("Preparing data_techexposure_company_portfolio.json.") prep_key_bars_portfolio( equity_results_portfolio = equity_results_portfolio, bonds_results_portfolio = bonds_results_portfolio, @@ -344,12 +365,14 @@ zip_outputs(dashboard_data_dir) zip_outputs <- function(dashboard_data_dir) { + log_debug("Preparing outputs zip archive.") json_filenames <- list.files(dashboard_data_dir, pattern = "[.]json$") zip_temp <- file.path(tempdir(), "zip_temp") dir.create(zip_temp, showWarnings = FALSE) for (json_filename in json_filenames) { + log_trace(paste("Adding", json_filename, "to zip archive.")) file.copy( from = file.path(dashboard_data_dir, json_filename), to = file.path(zip_temp, json_filename) @@ -363,6 +386,7 @@ zip_outputs <- function(dashboard_data_dir) { ) if (inherits(df, "data.frame")) { + log_trace(paste("Adding", csv_filename, "to zip archive.")) readr::write_csv( x = df, file = file.path(zip_temp, csv_filename), @@ -372,6 +396,7 @@ zip_outputs <- function(dashboard_data_dir) { } } + log_debug("Creating zip archive.") utils::zip( zipfile = file.path(dashboard_data_dir, "archive.zip"), files = list.files(zip_temp, full.names = TRUE), From dafd6aa8675a7dfba794e10b871a939b4ea7a1e4 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 11 Dec 2024 13:45:42 +0100 Subject: [PATCH 05/33] Git keep default dev output dir --- dashboard_output_dir/data/.gitkeep | 0 docker-compose.yml | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 dashboard_output_dir/data/.gitkeep diff --git a/dashboard_output_dir/data/.gitkeep b/dashboard_output_dir/data/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docker-compose.yml b/docker-compose.yml index 8960d0e..1bded95 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,9 +2,9 @@ services: workflow.pacta.dashboard: build: . - stdin_open: true - tty: true - entrypoint: ["R", "--args"] + # stdin_open: true + # tty: true + # entrypoint: ["R", "--args"] command: '{\"portfolio\": {\"files\": [\"default_portfolio.csv\"], \"holdingsDate\": \"2023-12-31\", \"name\": \"FooPortfolio\"}, \"inherit\": \"GENERAL_2023Q4\"}' environment: LOG_LEVEL: TRACE From 0ec48f22737b1c1ea148b7006e99bff8ec237356 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 11 Dec 2024 17:28:45 +0100 Subject: [PATCH 06/33] Output manifest and update param names --- R/build_dashboard.R | 35 ++++++++++++++++++++++++++++++++ R/prepare_pacta_dashboard_data.R | 4 ++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/R/build_dashboard.R b/R/build_dashboard.R index 4a511f7..b9cf777 100644 --- a/R/build_dashboard.R +++ b/R/build_dashboard.R @@ -33,4 +33,39 @@ build_dashboard <- function( ) } + out <- list( + input_files = c( + list.files( + dashboard_data_dir, + full.names = TRUE, + recursive = TRUE + ), + list.files( + dashboard_skeleton_files_dir, + full.names = TRUE, + recursive = TRUE + ), + list.files( + analysis_output_dir, + full.names = TRUE, + recursive = TRUE + ), + list.files( + benchmarks_dir, + full.names = TRUE, + recursive = TRUE + ) + ), + output_files = c( + list.files( + dashboard_output_dir, + full.names = TRUE, + recursive = TRUE + ) + ), + params = params + ) + + return(out) + } diff --git a/R/prepare_pacta_dashboard_data.R b/R/prepare_pacta_dashboard_data.R index b4ff84e..ee408d8 100644 --- a/R/prepare_pacta_dashboard_data.R +++ b/R/prepare_pacta_dashboard_data.R @@ -10,7 +10,7 @@ log_info("Preparing data for the PACTA dashboard.") # portfolio/user parameters ---------------------------------------------------- log_debug("Reading portfolio/user parameters.") -investor_name <- params[["user"]][["investor_name"]] +investor_name <- params[["user"]][["name"]] portfolio_name <- params[["portfolio"]][["name"]] peer_group <- params[["user"]][["peerGroup"]] language_select <- params[["user"]][["languageSelect"]] @@ -18,7 +18,7 @@ language_select <- params[["user"]][["languageSelect"]] currency_exchange_value <- params[["user"]][["currencyExchangeValue"]] display_currency <- params[["user"]][["displayCurrency"]] -select_scenario_other <- params[["reporting"]][["selectOther"]] +select_scenario_other <- params[["reporting"]][["scenarioOther"]] select_scenario <- params[["reporting"]][["selectScenario"]] green_techs <- params[["reporting"]][["greenTechs"]] From 0caccfd9cfeed2371053bdf3a0583269af25db7f Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 12:21:20 +0100 Subject: [PATCH 07/33] Add function to run full workflow --- R/run_dashboard_workflow.R | 87 +++++++++++++++++++ inst/extdata/scripts/prepare_dashboard_data.R | 2 +- 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 R/run_dashboard_workflow.R diff --git a/R/run_dashboard_workflow.R b/R/run_dashboard_workflow.R new file mode 100644 index 0000000..6843fa2 --- /dev/null +++ b/R/run_dashboard_workflow.R @@ -0,0 +1,87 @@ +#' Run Dashboard workflow +#' +#' Reun steps required to prepare a PACTA analysis and dashboard +#' +#' @param params (`list`) A list of parameters to be used in the analysis and +#' dashboard process. See JSON Schema for details. +#' @param raw_params (`character`) Raw JSON string of parameters. +#' @param run_analysis (`logical`) Run the analysis process. +#' @param analysis_output_dir Directory containing the PACTA analysis results. +#' @param benchmarks_dir Directory containing the benchmark analysis results. +#' @param pacta_data_dir filepath: Directory with "pacta-data" +#' @param portfolio_dir filepath: Directory with portfolio files +#' @param real_estate_dir Directory containing real estate data. +#' @param dashboard_output_dir Directory where the dashboard will be +#' @param score_card_dir Directory containing score card data. +#' @param summary_output_dir Directory where the executive summary will be +#' @param survey_dir Directory containing survey data. +#' saved. +#' saved. +#' @return (invisible) `TRUE` if the workflow was successful. Primarily called +#' for side effect of rendering files. +#' @export +run_dashboard_workflow <- function( + params, + raw_params, + run_analysis = TRUE, + run_dashboard = TRUE, + analysis_output_dir = Sys.getenv("ANALYSIS_OUTPUT_DIR"), + benchmarks_dir = Sys.getenv("BENCHMARKS_DIR"), + pacta_data_dir = Sys.getenv("PACTA_DATA_DIR"), + portfolio_dir = Sys.getenv("PORTFOLIO_DIR"), + dashboard_output_dir = Sys.getenv("DASHBOARD_OUTPUT_DIR") +) { + + log_trace("preparing manifest paths.") + analysis_manifest_path <- file.path( + analysis_output_dir, + "manifest.json" + ) + log_trace("Analysis manifest path: ", analysis_manifest_path) + dashboard_manifest_path <- file.path( + dashboard_output_dir, + "manifest.json" + ) + log_trace("Dashboard manifest path: ", dashboard_manifest_path) + + if (run_analysis || !file.exists(analysis_manifest_path)) { + + log_trace("running analysis workflow.") + analysis_manifest_info <- workflow.pacta::run_pacta( + params = params, + pacta_data_dir = pacta_data_dir, + output_dir = analysis_output_dir, + portfolio_dir = portfolio_dir + ) + + log_trace("exporting analysis manifest.") + pacta.workflow.utils::export_manifest( + input_files = analysis_manifest_info[["input_files"]], + output_files = analysis_manifest_info[["output_files"]], + params = analysis_manifest_info[["params"]], + manifest_path = analysis_manifest_path, + raw_params = raw_params + ) + + } + + if (run_dashboard) { + + log_trace("running dashboard workflow.") + dashboard_manifest_info <- build_dashboard( + params = params + ) + + log_trace("exporting dashboard manifest.") + pacta.workflow.utils::export_manifest( + input_files = dashboard_manifest_info[["input_files"]], + output_files = dashboard_manifest_info[["output_files"]], + params = dashboard_manifest_info[["params"]], + manifest_path = dashboard_manifest_path, + raw_params = raw_params + ) + + } + return(invisible(TRUE)) +} + diff --git a/inst/extdata/scripts/prepare_dashboard_data.R b/inst/extdata/scripts/prepare_dashboard_data.R index bd4fdd5..8e44e86 100644 --- a/inst/extdata/scripts/prepare_dashboard_data.R +++ b/inst/extdata/scripts/prepare_dashboard_data.R @@ -24,7 +24,7 @@ params <- pacta.workflow.utils::parse_raw_params( force_array = c("portfolio", "files") ) -manifest_info <- workflow.pacta.dashboard:::build_dashboard( +manifest_info <- workflow.pacta.dashboard:::run_dashboard_workflow( params = params ) From c8d63d07c6b4f27853b66c835f8d011804df92e0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 13:04:56 +0100 Subject: [PATCH 08/33] Local build with pacta-data --- .Rbuildignore | 1 + .dockerignore | 2 + .gitignore | 1 + docker-compose.yml | 12 ++- inst/extdata/scripts/prepare_dashboard_data.R | 9 +- tests/config/default_2022Q4.json | 16 ++++ tests/config/default_2023Q4.json | 16 ++++ tests/config/full_params_2022Q4.json | 77 ++++++++++++++++ tests/config/full_params_2023Q4.json | 77 ++++++++++++++++ tests/portfolios/default_portfolio.csv | 91 +++++++++++++++++++ 10 files changed, 293 insertions(+), 9 deletions(-) create mode 100644 tests/config/default_2022Q4.json create mode 100644 tests/config/default_2023Q4.json create mode 100644 tests/config/full_params_2022Q4.json create mode 100644 tests/config/full_params_2023Q4.json create mode 100644 tests/portfolios/default_portfolio.csv diff --git a/.Rbuildignore b/.Rbuildignore index 722cfa5..d4b755f 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -7,3 +7,4 @@ ^\.Rproj\.user$ ^docker-compose.yml$ ^dashboard_output_dir +^pacta-data/ diff --git a/.dockerignore b/.dockerignore index 23afe46..7500a75 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,3 +11,5 @@ benchmarks_dir/**/* dashboard_data_dir dashboard_data_dir/**/* docker-compose.yml +pacta-data/**/* +^tests/**/* diff --git a/.gitignore b/.gitignore index 84720e4..a3aee08 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ analysis_output_dir/**/* benchmarks_dir/**/* dashboard_output_dir/**/* !dashboard_output_dir/data/.gitkeep +pacta-data/**/* diff --git a/docker-compose.yml b/docker-compose.yml index 1bded95..8c9edbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,17 @@ services: BENCHMARKS_DIR: "/mnt/benchmarks_dir" DASHBOARD_DATA_DIR: "/mnt/dashboard_output_dir/data" DASHBOARD_OUTPUT_DIR: "/mnt/dashboard_output_dir" + PACTA_DATA_DIR: "/mnt/pacta-data" + PORTFOLIO_DIR: "/mnt/portfolios" volumes: + - type: bind + source: ${data_dir:-./pacta-data/} + target: /mnt/pacta-data + read_only: true + - type: bind + source: ${input_dir:-./tests/portfolios/} + target: /mnt/portfolios + read_only: true - type: bind source: ${benchmarks_dir:-./benchmarks_dir} target: /mnt/benchmarks_dir/ @@ -20,7 +30,7 @@ services: - type: bind source: ${analysis_output_dir:-./analysis_output_dir} target: /mnt/analysis_output_dir - read_only: true + read_only: false - type: bind source: ${dashboard_data_dir:-./dashboard_output_dir/data} target: /mnt/dashboard_output_dir/data diff --git a/inst/extdata/scripts/prepare_dashboard_data.R b/inst/extdata/scripts/prepare_dashboard_data.R index 8e44e86..49aad2c 100644 --- a/inst/extdata/scripts/prepare_dashboard_data.R +++ b/inst/extdata/scripts/prepare_dashboard_data.R @@ -25,13 +25,6 @@ params <- pacta.workflow.utils::parse_raw_params( ) manifest_info <- workflow.pacta.dashboard:::run_dashboard_workflow( - params = params -) - -pacta.workflow.utils::export_manifest( - input_files = manifest_info[["input_files"]], - output_files = manifest_info[["output_files"]], - params = manifest_info[["params"]], - manifest_path = file.path(Sys.getenv("REPORT_OUTPUT_DIR"), "manifest.json"), + params = params, raw_params = raw_params ) diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json new file mode 100644 index 0000000..f9897b4 --- /dev/null +++ b/tests/config/default_2022Q4.json @@ -0,0 +1,16 @@ +{ + "holdingsDate": "2022Q4", + "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2022Q4/output_dir", + "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", + "parameters": { + "portfolio": { + "holdingsDate": "2023-12-31", + "files": [ + "default_portfolio.csv" + ], + "name": "Default Portfolio" + }, + "inherit": "GENERAL_2022Q4" + } +} diff --git a/tests/config/default_2023Q4.json b/tests/config/default_2023Q4.json new file mode 100644 index 0000000..4843e13 --- /dev/null +++ b/tests/config/default_2023Q4.json @@ -0,0 +1,16 @@ +{ + "holdingsDate": "2023Q4", + "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2023Q4/output_dir", + "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", + "parameters": { + "portfolio": { + "holdingsDate": "2023-12-31", + "files": [ + "default_portfolio.csv" + ], + "name": "Default Portfolio" + }, + "inherit": "GENERAL_2023Q4" + } +} diff --git a/tests/config/full_params_2022Q4.json b/tests/config/full_params_2022Q4.json new file mode 100644 index 0000000..ddefced --- /dev/null +++ b/tests/config/full_params_2022Q4.json @@ -0,0 +1,77 @@ +{ + "holdingsDate": "2022Q4", + "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2022Q4/output_dir", + "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", + "parameters": { + "portfolio": { + "holdingsDate": "2023-12-31", + "files": [ + "default_portfolio.csv" + ], + "name": "Default Portfolio" + }, + "analysis": { + "equityMarketList": [ + "GlobalMarket", + "DevelopedMarket", + "EmergingMarket" + ], + "scenarioGeographiesList": [ + "Global", + "GlobalAggregate", + "NonOECD", + "OECD" + ], + "scenarioSourcesList": [ + "GECO2022", + "ISF2021", + "WEO2022" + ], + "sectorList": [ + "Power", + "Automotive", + "Coal", + "Steel", + "Aviation", + "Cement" + ], + "startYear": 2023, + "timeHorizon": 5 + }, + "user": { + "name": "Default Investor", + "languageSelect": "EN", + "peerGroup": "bank", + "currencyExchangeValue": 1, + "displayCurrency": "USD" + }, + "reporting": { + "projectCode": "GENERAL", + "projectReportName": "general", + "greenTechs": [ + "RenewablesCap", + "HydroCap", + "NuclearCap", + "Hybrid", + "Electric", + "FuelCell", + "Electric Arc Furnace" + ], + "pactaSectorsNotAnalysed": [ + "Steel", + "Aviation", + "Cement" + ], + "portfolioAllocationMethod": "portfolio_weight", + "scenarioGeography": "Global", + "scenarioOther": "GECO2022_1.5C", + "selectScenario": "WEO2022_NZE_2050", + "techRoadmapSectors": [ + "Power", + "Automotive", + "Coal" + ] + } + } +} diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json new file mode 100644 index 0000000..cdf0c6d --- /dev/null +++ b/tests/config/full_params_2023Q4.json @@ -0,0 +1,77 @@ +{ + "holdingsDate": "2023Q4", + "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/full_params_2023Q4/output_dir", + "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", + "parameters": { + "portfolio": { + "holdingsDate": "2023-12-31", + "files": [ + "default_portfolio.csv" + ], + "name": "Default Portfolio" + }, + "analysis": { + "equityMarketList": [ + "GlobalMarket", + "DevelopedMarket", + "EmergingMarket" + ], + "scenarioGeographiesList": [ + "Global", + "GlobalAggregate", + "NonOECD", + "OECD" + ], + "scenarioSourcesList": [ + "GECO2023", + "ISF2023", + "WEO2023" + ], + "sectorList": [ + "Power", + "Automotive", + "Coal", + "Steel", + "Aviation", + "Cement" + ], + "startYear": 2023, + "timeHorizon": 5 + }, + "user": { + "name": "Default Investor", + "languageSelect": "EN", + "peerGroup": "bank", + "currencyExchangeValue": 1, + "displayCurrency": "USD" + }, + "reporting": { + "projectCode": "GENERAL", + "projectReportName": "general", + "greenTechs": [ + "RenewablesCap", + "HydroCap", + "NuclearCap", + "Hybrid", + "Electric", + "FuelCell", + "Electric Arc Furnace" + ], + "pactaSectorsNotAnalysed": [ + "Steel", + "Aviation", + "Cement" + ], + "portfolioAllocationMethod": "portfolio_weight", + "scenarioGeography": "Global", + "scenarioOther": "GECO2022_1.5C", + "selectScenario": "WEO2022_NZE_2050", + "techRoadmapSectors": [ + "Power", + "Automotive", + "Coal" + ] + } + } +} diff --git a/tests/portfolios/default_portfolio.csv b/tests/portfolios/default_portfolio.csv new file mode 100644 index 0000000..de58d78 --- /dev/null +++ b/tests/portfolios/default_portfolio.csv @@ -0,0 +1,91 @@ +isin,market_value,currency +XS1297557172,6821206,USD +DE000A1R07L4,4724723,USD +US345370BV11,5619252,USD +CND10001QD60,1124438,USD +XS2356040357,3076756,USD +US31428XAY22,8235607,USD +FR0012650281,641235,USD +FR001400F2Q0,4190307,USD +USQ94606AE22,4118654,USD +US911312BB15,5414864,USD +INE331A07273,2972779,USD +INE481G07166,3244497,USD +INE010A08065,4488737,USD +US86614RAF47,5793655,USD +FR0124261621,1581767,USD +US60937CAA36,900439,USD +US93317QAD79,9993273,USD +TH0148034703,8657460,USD +XS2325566847,9459870,USD +US02076XAC65,3952496,USD +US880779BA01,3506143,USD +USU88125AC04,3401211,USD +INE208A07281,8670483,USD +US69373UAA51,267878,USD +US12594KAA07,4094599,USD +US205677AB38,3309593,USD +US69370RAK32,3519158,USD +US707887A@47,1577541,USD +US166764AA86,6575752,USD +US71654QCP54,7084677,USD +INE206D08162,6210214,USD +US745310AB85,709698,USD +CA44882ZXZ12,2809584,USD +JP360540AR10,9641731,USD +CA44885ZEQ96,9683822,USD +INE017A08235,3910673,USD +XS1555576641,2598631,USD +XS1381693248,2117715,USD +NO0010892870,8899756,USD +NO0010806912,7280862,USD +USU85795AQ29,2560264,USD +JP338603AA35,6806168,USD +DE000A2GESQ1,1567613,USD +INE114A07802,8605761,USD +US03938LAA26,2698943,USD +KR7005380001,9338795,USD +CNE100000Q35,1068423,USD +KYG3777B1032,9637388,USD +US5787871038,5467105,USD +SE0016844831,4168401,USD +TW0002646007,6371201,USD +INE646L01027,8355228,USD +US38045R2067,7952384,USD +PHY1234G1032,180886,USD +US00910M1009,6809079,USD +INE070A01015,1013567,USD +CNE000000XV4,8873793,USD +PK0059501012,5829360,USD +INE583C01021,6059486,USD +MYL5000OO004,6433976,USD +AU000000BCB5,9186193,USD +US03940R1077,265189,USD +ID1000162902,1712519,USD +US03485P3001,4712749,USD +CNE0000017V2,5989615,USD +NL0010545661,3343735,USD +CNE000000FH0,8915246,USD +US6937181088,9800223,USD +US9288541082,7226287,USD +CA62910L1022,9449577,USD +IL0011419699,7746343,USD +SE0020180917,1788562,USD +AU000000BUY9,1578448,USD +KR7096770003,860209,USD +US45325E2019,4071578,USD +CA45790B7088,8510107,USD +US0188021085,7030638,USD +JP3981200003,344741,USD +GB00B1VNSX38,6493591,USD +BRENGICDAM16,9257903,USD +MHY271836006,1571628,USD +MHY2687W1241,2627572,USD +TH0363010R10,8711798,USD +JP3247600004,4447848,USD +IL0065100930,1121308,USD +QA000A0KD6K3,6405719,USD +US92023R4074,8204370,USD +JP3342000001,8564047,USD +US88031M1099,149263,USD +TW0009957001,4538558,USD From d6beb738350b6e1008b2bf33132d4eb2e92f29b0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 13:10:03 +0100 Subject: [PATCH 09/33] disable GH-pages preview --- .github/workflows/docker.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ae813e7..2f55219 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,10 +21,10 @@ jobs: with: do-check-r-sysdeps: false - gh-pages: - name: gh-pages - uses: ./.github/workflows/gh-pages.yml - needs: [docker] - secrets: inherit - with: - full-image-name: ${{ needs.docker.outputs.full-image-name }} + # gh-pages: + # name: gh-pages + # uses: ./.github/workflows/gh-pages.yml + # needs: [docker] + # secrets: inherit + # with: + # full-image-name: ${{ needs.docker.outputs.full-image-name }} From f69fd9f8bf999163b0c777c776f5f022b5a8a2ac Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 14:03:31 +0100 Subject: [PATCH 10/33] Add GH test workflow and update configs --- .github/workflows/docker.yml | 17 +++ .github/workflows/test.yml | 196 +++++++++++++++++++++++++++ tests/config/default_2022Q4.json | 1 - tests/config/default_2023Q4.json | 1 - tests/config/full_params_2022Q4.json | 1 - tests/config/full_params_2023Q4.json | 1 - 6 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2f55219..fda8b55 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,6 +21,23 @@ jobs: with: do-check-r-sysdeps: false + test: + name: Test + uses: ./.github/workflows/test.yml + needs: [docker] + secrets: inherit + strategy: + fail-fast: false + matrix: + config-name: + - default_2022Q4 + - default_2023Q4 + - full_params_2022Q4 + - full_params_2023Q4 + with: + full-image-name: ${{ needs.docker.outputs.full-image-name }} + config-name: ${{ matrix.config-name }} + # gh-pages: # name: gh-pages # uses: ./.github/workflows/gh-pages.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a39a7a7 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,196 @@ +--- +name: Test docker image + +on: + workflow_call: + inputs: + full-image-name: + required: true + type: string + config-name: + required: true + type: string + results-url: + description: azure blob store path for results + required: false + default: "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-webapp-results" + type: string + reports-url: + description: azure blob store path for results + required: false + default: "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-webapp-results-reports" + type: string + +jobs: + test: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + + - name: Checkout workflow + uses: actions/checkout@v4 + + - name: Prepare environment + id: prepare + env: + CONFIG_NAME: ${{ inputs.config-name }} + GITHUB_REF_NAME: ${{ github.ref_name}} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + WORKSPACE: ${{ github.workspace }} + run: | + config_file="tests/config/$CONFIG_NAME.json" + echo "config_file: $config_file" + + HOLDINGS_DATE="$(jq -rc '.holdingsDate' $config_file)" + echo "holdings-date=$HOLDINGS_DATE" + echo "holdings-date=$HOLDINGS_DATE" >> "$GITHUB_OUTPUT" + + PACTA_DATA_URL="$(jq -rc '.pactaDataURL' $config_file)" + echo "pacta-data-url=$PACTA_DATA_URL" + echo "pacta-data-url=$PACTA_DATA_URL" >> "$GITHUB_OUTPUT" + + BENCHMARKS_URL="$(jq -rc '.benchmarksURL' $config_file)" + echo "benchmarks-url=$BENCHMARKS_URL" + echo "benchmarks-url=$BENCHMARKS_URL" >> "$GITHUB_OUTPUT" + + # includes handling for null/missing keys + PARAMETERS="$(jq -rc '.parameters | select( . != null )' $config_file)" + echo "parameters=$PARAMETERS" + echo "parameters=$PARAMETERS" >> "$GITHUB_OUTPUT" + + TEST_DIR="$WORKSPACE/$GITHUB_REF_NAME/$GITHUB_RUN_NUMBER/$GITHUB_RUN_ATTEMPT/$CONFIG_NAME" + mkdir -p $TEST_DIR + echo "test-dir=$TEST_DIR" + echo "test-dir=$TEST_DIR" >> "$GITHUB_OUTPUT" + + DASHBOARD_OUTPUT_DIR="$TEST_DIR/dashboard_output_dir" + mkdir -p "$DASHBOARD_OUTPUT_DIR" + chmod -R 777 "$DASHBOARD_OUTPUT_DIR" + echo "dashboard-output-dir=$DASHBOARD_OUTPUT_DIR" + echo "dashboard-output-dir=$DASHBOARD_OUTPUT_DIR" >> "$GITHUB_OUTPUT" + + DASHBOARD_DATA_DIR="$TEST_DIR/$DASHBOARD_OUTPUT_DIR/data" + mkdir -p "$DASHBOARD_DATA_DIR" + chmod -R 777 "$DASHBOARD_DATA_DIR" + echo "dashboard-data-dir=$DASHBOARD_DATA_DIR" + echo "dashboard-data-dir=$DASHBOARD_DATA_DIR" >> "$GITHUB_OUTPUT" + + ANALYSIS_OUTPUT_DIR="$TEST_DIR/analysis_output_dir" + mkdir -p "$ANALYSIS_OUTPUT_DIR" + chmod -R 777 "$ANALYSIS_OUTPUT_DIR" + echo "analysis-output-dir=$ANALYSIS_OUTPUT_DIR" + echo "analysis-output-dir=$ANALYSIS_OUTPUT_DIR" >> "$GITHUB_OUTPUT" + + # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + - name: Download pacta-data + id: download-pacta-data + uses: RMI-PACTA/actions/actions/azure/blob-copy@main + with: + source: ${{ steps.prepare.outputs.pacta-data-url }} + destination: tmp_benchmarks_dir + + - name: Download Benchmarks + id: download-benchmarks + uses: RMI-PACTA/actions/actions/azure/blob-copy@main + with: + source: ${{ steps.prepare.outputs.benchmarks-url }} + destination: tmp_benchmarks_dir + + - name: Show TEST_DIR + id: ls-test-dir + env: + TEST_DIR: ${{ steps.prepare.outputs.test-dir }} + run: | + ls -laR "$TEST_DIR" + + - name: Run Docker Image + env: + FULL_IMAGE_NAME: ${{ inputs.full-image-name }} + WORKSPACE: ${{ github.workspace }} + PARAMETERS: ${{ steps.prepare.outputs.parameters }} + ANALYSIS_OUTPUT_DIR: ${{ steps.prepare.outputs.analysis-output-dir }} + BENCHMARKS_DIR: benchmarks_dir + PACTA_DATA_DIR: pacta-data + PORTFOLIO_DIR: tests/portfolios + DASHBOARD_DATA_DIR: ${{ steps.prepare.outputs.dashboard-data-dir }} + DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} + run: | + + docker run \ + --network none \ + --env LOG_LEVEL=TRACE \ + --env ANALYSIS_OUTPUT_DIR="/mnt/analysis_output_dir" \ + --env OUTPUT_DIR="/mnt/analysis_output_dir" \ + --env BENCHMARKS_DIR="/mnt/benchmarks_dir" \ + --env PACTA_DATA_DIR="/mnt/pacta-data" \ + --env PORTFOLIO_DIR="/mnt/portfolios" \ + --env DASHBOARD_OUTPUT_DIR="/mnt/dashboard_output_dir" \ + --env DASHBOARD_DATA_DIR="/mnt/dashboard_output_dir/data" \ + --mount type=bind,readonly,source=${WORKSPACE}/${BENCHMARKS_DIR},target=/mnt/benchmarks_dir \ + --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ + --mount type=bind,readonly,source=${WORKSPACE}/${PORTFOLIO_DIR},target=/mnt/portfolios \ + --mount type=bind,source=${ANALYSIS_OUTPUT_DIR},target=/mnt/analysis_output_dir \ + --mount type=bind,source=${DASHBOARD_OUTPUT_DIR},target=/mnt/dashboard_output_dir \ + --mount type=bind,source=${DASHBOARD_DATA_DIR},target=/mnt/dashboard_output_dir/data \ + $FULL_IMAGE_NAME \ + "$PARAMETERS" + + - name: List outputs + env: + DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} + run: | + ls -lR $DASHBOARD_OUTPUT_DIR + + # - name: Prepare comment artifact + # id: prepare-artifact + # env: + # commit_time: ${{ github.event.pull_request.updated_at }} + # config_name: ${{ inputs.config-name }} + # full_image_name: ${{ inputs.full-image-name }} + # git_sha: ${{ github.event.pull_request.head.sha }} + # report_url: ${{ steps.export-outputs.outputs.report-url }} + # summary_url: ${{ steps.export-outputs.outputs.summary-url }} + # run: | + # mkdir -p /tmp/comment-json + # json_filename="$( \ + # echo "report-comment-json-merge-$full_image_name-$config_name" | \ + # tr '/:._' '-' \ + # ).json" + # echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" + # json_file="/tmp/comment-json/$json_filename" + # jq \ + # -n \ + # --arg commit_time "$commit_time" \ + # --arg config_name "$config_name" \ + # --arg git_sha "$git_sha" \ + # --arg report_url "$report_url" \ + # --arg summary_url "$summary_url" \ + # --arg full_image_name "$full_image_name" \ + # '{ + # "commit_time": $commit_time, + # "git_sha": $git_sha, + # "config": $config_name, + # "report": ("[Report](" + $report_url + ")"), + # "summary": (if (("" + $summary_url) == "") then null else "[Summary](" + $summary_url + ")" end), + # "image": ("`" + $full_image_name + "`") + # }' \ + # >> $json_file + # cat "$json_file" + + # - name: Upload comment JSON + # uses: actions/upload-artifact@v4 + # with: + # name: ${{ steps.prepare-artifact.outputs.json-filename }} + # path: /tmp/comment-json/* + # if-no-files-found: error + # retention-days: 1 diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json index f9897b4..868a6f0 100644 --- a/tests/config/default_2022Q4.json +++ b/tests/config/default_2022Q4.json @@ -1,7 +1,6 @@ { "holdingsDate": "2022Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", - "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2022Q4/output_dir", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", "parameters": { "portfolio": { diff --git a/tests/config/default_2023Q4.json b/tests/config/default_2023Q4.json index 4843e13..be9ebbd 100644 --- a/tests/config/default_2023Q4.json +++ b/tests/config/default_2023Q4.json @@ -1,7 +1,6 @@ { "holdingsDate": "2023Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", - "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2023Q4/output_dir", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", "parameters": { "portfolio": { diff --git a/tests/config/full_params_2022Q4.json b/tests/config/full_params_2022Q4.json index ddefced..fd0a8ed 100644 --- a/tests/config/full_params_2022Q4.json +++ b/tests/config/full_params_2022Q4.json @@ -1,7 +1,6 @@ { "holdingsDate": "2022Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", - "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2022Q4/output_dir", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", "parameters": { "portfolio": { diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json index cdf0c6d..909af2b 100644 --- a/tests/config/full_params_2023Q4.json +++ b/tests/config/full_params_2023Q4.json @@ -1,7 +1,6 @@ { "holdingsDate": "2023Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", - "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/full_params_2023Q4/output_dir", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", "parameters": { "portfolio": { From 45f01e90eded11711c6dc1f682926f5458c81bd6 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 14:22:24 +0100 Subject: [PATCH 11/33] read pacta-data and benchmarks from blob, not file share --- tests/config/default_2022Q4.json | 4 ++-- tests/config/default_2023Q4.json | 4 ++-- tests/config/full_params_2022Q4.json | 4 ++-- tests/config/full_params_2023Q4.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json index 868a6f0..b2827b2 100644 --- a/tests/config/default_2022Q4.json +++ b/tests/config/default_2022Q4.json @@ -1,7 +1,7 @@ { "holdingsDate": "2022Q4", - "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", - "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", + "pactaDataURL": "https://pactadatadev.blob.core.windows.net/pacta-data-webapp/2022Q4/2022Q4_20240426T113151Z", + "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2022Q4/2022Q4_20240529T002407Z", "parameters": { "portfolio": { "holdingsDate": "2023-12-31", diff --git a/tests/config/default_2023Q4.json b/tests/config/default_2023Q4.json index be9ebbd..7b65a21 100644 --- a/tests/config/default_2023Q4.json +++ b/tests/config/default_2023Q4.json @@ -1,7 +1,7 @@ { "holdingsDate": "2023Q4", - "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", - "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", + "pactaDataURL": "https://pactadatadev.blob.core.windows.net/pacta-data-webapp/2023Q4/2023Q4_20240424T120055Z", + "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2023Q4/2023Q4_20240529T002355Z", "parameters": { "portfolio": { "holdingsDate": "2023-12-31", diff --git a/tests/config/full_params_2022Q4.json b/tests/config/full_params_2022Q4.json index fd0a8ed..c25ec83 100644 --- a/tests/config/full_params_2022Q4.json +++ b/tests/config/full_params_2022Q4.json @@ -1,7 +1,7 @@ { "holdingsDate": "2022Q4", - "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", - "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", + "pactaDataURL": "https://pactadatadev.blob.core.windows.net/pacta-data-webapp/2022Q4/2022Q4_20240426T113151Z", + "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2022Q4/2022Q4_20240529T002407Z", "parameters": { "portfolio": { "holdingsDate": "2023-12-31", diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json index 909af2b..a232958 100644 --- a/tests/config/full_params_2023Q4.json +++ b/tests/config/full_params_2023Q4.json @@ -1,7 +1,7 @@ { "holdingsDate": "2023Q4", - "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", - "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", + "pactaDataURL": "https://pactadatadev.blob.core.windows.net/pacta-data-webapp/2023Q4/2023Q4_20240424T120055Z", + "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2023Q4/2023Q4_20240529T002355Z", "parameters": { "portfolio": { "holdingsDate": "2023-12-31", From 22b3cd89f4962525f7c01038ed9d3d1efe2a3d2d Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 14:57:56 +0100 Subject: [PATCH 12/33] Move pactadata and benchmarks to correct directories --- .github/workflows/test.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a39a7a7..58a9058 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,16 +10,6 @@ on: config-name: required: true type: string - results-url: - description: azure blob store path for results - required: false - default: "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-webapp-results" - type: string - reports-url: - description: azure blob store path for results - required: false - default: "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-webapp-results-reports" - type: string jobs: test: @@ -97,7 +87,7 @@ jobs: uses: RMI-PACTA/actions/actions/azure/blob-copy@main with: source: ${{ steps.prepare.outputs.pacta-data-url }} - destination: tmp_benchmarks_dir + destination: tmp_pacta_data_dir - name: Download Benchmarks id: download-benchmarks @@ -111,6 +101,10 @@ jobs: env: TEST_DIR: ${{ steps.prepare.outputs.test-dir }} run: | + mkdir -p pacta-data + mv tmp_pacta_data_dir/*/*/* pacta-data + mkdir -p benchmarks_dir + mv tmp_benchmarks_dir/*/*/* benchmarks_dir ls -laR "$TEST_DIR" - name: Run Docker Image From a3229dae873564d30e31c3dcb554d8e73a308dc4 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 15:14:57 +0100 Subject: [PATCH 13/33] Fix path (un-concat 2 absolute paths) --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 58a9058..b5fa5ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,7 +62,7 @@ jobs: echo "dashboard-output-dir=$DASHBOARD_OUTPUT_DIR" echo "dashboard-output-dir=$DASHBOARD_OUTPUT_DIR" >> "$GITHUB_OUTPUT" - DASHBOARD_DATA_DIR="$TEST_DIR/$DASHBOARD_OUTPUT_DIR/data" + DASHBOARD_DATA_DIR="$DASHBOARD_OUTPUT_DIR/data" mkdir -p "$DASHBOARD_DATA_DIR" chmod -R 777 "$DASHBOARD_DATA_DIR" echo "dashboard-data-dir=$DASHBOARD_DATA_DIR" From b2ad02b70e40e0c7363d0a815150baa206a0ed5d Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 15:29:53 +0100 Subject: [PATCH 14/33] Upload built dashboard as artifact --- .github/workflows/test.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b5fa5ed..b932b55 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -141,9 +141,18 @@ jobs: - name: List outputs env: + CONFIG_NAME: ${{ inputs.config-name }} DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} run: | ls -lR $DASHBOARD_OUTPUT_DIR + cp -r $DASHBOARD_OUTPUT_DIR $CONFIG_NAME + + - name: Upload built dashboard as artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.prepare-artifact.outputs.json-filename }} + path: ${{ inputs.config-name }} + if-no-files-found: error # - name: Prepare comment artifact # id: prepare-artifact From 79b67634fca67db1ec7afdde59cae5f5029de1b1 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 16:02:48 +0100 Subject: [PATCH 15/33] tar artifact before upload --- .github/workflows/test.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b932b55..146c1f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -141,17 +141,23 @@ jobs: - name: List outputs env: - CONFIG_NAME: ${{ inputs.config-name }} DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} run: | ls -lR $DASHBOARD_OUTPUT_DIR - cp -r $DASHBOARD_OUTPUT_DIR $CONFIG_NAME + + - name: prepare artifact + env: + CONFIG_NAME: ${{ inputs.config-name }} + DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} + run: | + tar -cvf "$CONFIG_NAME.tar.gz" "$DASHBOARD_OUTPUT_DIR" + - name: Upload built dashboard as artifact uses: actions/upload-artifact@v4 with: - name: ${{ steps.prepare-artifact.outputs.json-filename }} - path: ${{ inputs.config-name }} + name: dashboard-build-${{ inputs.config-name }} + path: ${{ inputs.config-name }}.tar.gz if-no-files-found: error # - name: Prepare comment artifact From 4dfc1f22776fdf64803f2da861c72c8ba70c753d Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 16:39:57 +0100 Subject: [PATCH 16/33] Reenable GH-pages preview --- .github/workflows/docker.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fda8b55..6244c95 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -38,10 +38,10 @@ jobs: full-image-name: ${{ needs.docker.outputs.full-image-name }} config-name: ${{ matrix.config-name }} - # gh-pages: - # name: gh-pages - # uses: ./.github/workflows/gh-pages.yml - # needs: [docker] - # secrets: inherit - # with: - # full-image-name: ${{ needs.docker.outputs.full-image-name }} + gh-pages: + name: gh-pages + uses: ./.github/workflows/gh-pages.yml + needs: [docker, test] + secrets: inherit + with: + full-image-name: ${{ needs.docker.outputs.full-image-name }} From a97ac8da69a3c069259a5dc0c2c5d72abc5dba1c Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 16:53:13 +0100 Subject: [PATCH 17/33] Update gh pasges action --- .github/workflows/gh-pages.yml | 67 +++------------------------------- 1 file changed, 6 insertions(+), 61 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 6eeb621..eef853d 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -21,68 +21,13 @@ jobs: - name: Checkout uses: actions/checkout@v3 - # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended - - name: Azure Login - uses: azure/login@v2 + - uses: actions/download-artifact@v4 with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + pattern: dashboard-build- + merge-multiple: true + path: extracted_builds - - name: Download Analysis Outputs - id: download-analysis-outputs - uses: RMI-PACTA/actions/actions/azure/blob-copy@main - with: - source: https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/main/latest/full_params_2023Q4/analysis_output_dir - destination: tmp_analysis_output_dir - - - name: Download Benchmarks - id: download-benchmarks - uses: RMI-PACTA/actions/actions/azure/blob-copy@main - with: - source: https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2023Q4/2023Q4_20240529T002355Z - destination: tmp_benchmarks_dir - - - name: move output and analysis files to correct directory - run: | - mv tmp_analysis_output_dir/main/latest/full_params_2023Q4/analysis_output_dir analysis_output_dir - ls analysis_output_dir - mv tmp_benchmarks_dir/2023Q4/2023Q4_20240529T002355Z benchmarks_dir - ls benchmarks_dir - - - name: Create Output Directories - run: | - mkdir -p dashboard_output_dir/data - - - name: Run Docker Image - env: - FULL_IMAGE_NAME: ${{ inputs.full-image-name }} - WORKSPACE: ${{ github.workspace }} - PARAMETERS: "" - ANALYSIS_OUTPUT_DIR: "analysis_output_dir" - BENCHMARKS_DIR: "benchmarks_dir" - DASHBOARD_DATA_DIR: "dashboard_output_dir/data" - DASHBOARD_OUTPUT_DIR: "dashboard_output_dir" - run: | - - docker run \ - --network none \ - --user $(id -u):$(id -g) \ - --env LOG_LEVEL=TRACE \ - --env ANALYSIS_OUTPUT_DIR="/mnt/analysis_output_dir" \ - --env BENCHMARKS_DIR="/mnt/benchmarks_dir" \ - --env DASHBOARD_DATA_DIR="/mnt/dashboard_output_dir/data" \ - --env DASHBOARD_OUTPUT_DIR="/mnt/dashboard_output_dir" \ - --mount type=bind,readonly,source=${WORKSPACE}/${BENCHMARKS_DIR},target=/mnt/benchmarks_dir \ - --mount type=bind,source=${WORKSPACE}/${ANALYSIS_OUTPUT_DIR},target=/mnt/analysis_output_dir \ - --mount type=bind,source=${WORKSPACE}/${DASHBOARD_OUTPUT_DIR},target=/mnt/dashboard_output_dir \ - --mount type=bind,source=${WORKSPACE}/${DASHBOARD_DATA_DIR},target=/mnt/dashboard_output_dir/data \ - $FULL_IMAGE_NAME \ - "$PARAMETERS" - - - name: chmod & List outputs - run: | - ls -lR dashboard_output_dir + - run: ls -laR extracted_builds - name: Add nojekyll file if: github.event.action != 'closed' @@ -103,7 +48,7 @@ jobs: if: ${{ github.event_name == 'pull_request' }} uses: rossjrw/pr-preview-action@v1 with: - source-dir: dashboard_output_dir + source-dir: extracted_builds preview-branch: gh-pages umbrella-dir: pr-preview action: auto From 61b2f72749e650dafa41e76a5eb605259990004e Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 17:05:57 +0100 Subject: [PATCH 18/33] Add wildcard to artifact pattern --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index eef853d..00ac311 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/download-artifact@v4 with: - pattern: dashboard-build- + pattern: dashboard-build-* merge-multiple: true path: extracted_builds From bd6038556ac8b558653aa25d2f3befc2f07ba41b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 12 Dec 2024 17:50:45 +0100 Subject: [PATCH 19/33] Don't store full dir structure in tarfile --- .github/workflows/gh-pages.yml | 9 +++++++++ .github/workflows/test.yml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 00ac311..2aee191 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -27,6 +27,14 @@ jobs: merge-multiple: true path: extracted_builds + - name: extract tarfiles + run: | + for tarfile in *.tar.gz + do + tar -xvzf "$tarfile" + done + + - run: ls -laR extracted_builds - name: Add nojekyll file @@ -52,3 +60,4 @@ jobs: preview-branch: gh-pages umbrella-dir: pr-preview action: auto + dry-run: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 146c1f2..68e1869 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -150,7 +150,7 @@ jobs: CONFIG_NAME: ${{ inputs.config-name }} DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} run: | - tar -cvf "$CONFIG_NAME.tar.gz" "$DASHBOARD_OUTPUT_DIR" + tar -cv -f "$CONFIG_NAME.tar.gz" -C "$DASHBOARD_OUTPUT_DIR" . - name: Upload built dashboard as artifact From e06e061316e0c1fee762e422a488efafc754fcaf Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 10:33:41 +0100 Subject: [PATCH 20/33] Increment version number to 0.0.0.9008 --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 6478d8f..ac4a0f4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: workflow.pacta.dashboard Title: Run PACTA dashboard JSON generation -Version: 0.0.0.9007 +Version: 0.0.0.9008 Authors@R: c(person(given = "Alex", family = "Axthelm", From 011f67f97e27c9d4500aab099a0a21d7e5438c3d Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 10:37:33 +0100 Subject: [PATCH 21/33] Let GH action handle making archive --- .github/workflows/test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68e1869..c5fe506 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,19 +145,19 @@ jobs: run: | ls -lR $DASHBOARD_OUTPUT_DIR - - name: prepare artifact - env: - CONFIG_NAME: ${{ inputs.config-name }} - DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} - run: | - tar -cv -f "$CONFIG_NAME.tar.gz" -C "$DASHBOARD_OUTPUT_DIR" . + # - name: prepare artifact + # env: + # CONFIG_NAME: ${{ inputs.config-name }} + # DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} + # run: | + # tar -cv -f "$CONFIG_NAME.tar.gz" -C "$DASHBOARD_OUTPUT_DIR" . - name: Upload built dashboard as artifact uses: actions/upload-artifact@v4 with: name: dashboard-build-${{ inputs.config-name }} - path: ${{ inputs.config-name }}.tar.gz + path: ${{ steps.prepare.outputs.dashboard-output-dir }} if-no-files-found: error # - name: Prepare comment artifact From b850c5d85f2acf584493560e78672ab64daac6b3 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 10:49:16 +0100 Subject: [PATCH 22/33] Show files (debug step) --- .github/workflows/gh-pages.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 2aee191..15740ad 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -27,12 +27,16 @@ jobs: merge-multiple: true path: extracted_builds - - name: extract tarfiles + - name: Show files run: | - for tarfile in *.tar.gz - do - tar -xvzf "$tarfile" - done + ls -laR extracted_builds + + # - name: extract tarfiles + # run: | + # for tarfile in *.tar.gz + # do + # tar -xvzf "$tarfile" + # done - run: ls -laR extracted_builds From 50aede36622c1679b9a52bc7904f7dc426dfc129 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 11:09:05 +0100 Subject: [PATCH 23/33] Do not merge multiple artifacts --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 15740ad..ed115a9 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/download-artifact@v4 with: pattern: dashboard-build-* - merge-multiple: true + merge-multiple: false path: extracted_builds - name: Show files From 9c666b3d4730ad4457bc62eaecf9b84756b842c0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 12:22:08 +0100 Subject: [PATCH 24/33] Add (very simple) landing pages for PR preview --- .github/workflows/gh-pages.yml | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index ed115a9..615826d 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -27,19 +27,29 @@ jobs: merge-multiple: false path: extracted_builds - - name: Show files + - name: Generate PR landing page + working-directory: extracted_builds + shell: bash run: | - ls -laR extracted_builds - - # - name: extract tarfiles - # run: | - # for tarfile in *.tar.gz - # do - # tar -xvzf "$tarfile" - # done + echo "<\!DOCTYPE html>" > index.html + for target_page in $(find -f */index.html) + do + echo "$target_page" + link_text="$( \ + echo "$target_page" | \ + sed 's/^dashboard-build-//' | \ + sed 's/\/index\.html//' \ + )" + echo "$link_text" + echo "$link_text
" >> index.html + done + echo "" >> index.html + cat index.html - - run: ls -laR extracted_builds + - name: Show files + run: | + ls -laR extracted_builds - name: Add nojekyll file if: github.event.action != 'closed' From e8bd34fa8b658d031562b6708858953424482821 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 12:38:11 +0100 Subject: [PATCH 25/33] update pr landing page generation --- .github/workflows/gh-pages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 615826d..64b3e9f 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -32,8 +32,8 @@ jobs: shell: bash run: | - echo "<\!DOCTYPE html>" > index.html - for target_page in $(find -f */index.html) + echo "" > index.html + for target_page in $(find */index.html) do echo "$target_page" link_text="$( \ From f1c86b826dd995861fbd6a87cc0eea8198ec9f62 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 14:10:31 +0100 Subject: [PATCH 26/33] Apply suggestions from code review: Update dates for 22Q4 Co-authored-by: Jackson Hoffart --- tests/config/default_2022Q4.json | 2 +- tests/config/full_params_2022Q4.json | 4 ++-- tests/config/full_params_2023Q4.json | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json index b2827b2..12e6456 100644 --- a/tests/config/default_2022Q4.json +++ b/tests/config/default_2022Q4.json @@ -4,7 +4,7 @@ "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2022Q4/2022Q4_20240529T002407Z", "parameters": { "portfolio": { - "holdingsDate": "2023-12-31", + "holdingsDate": "2022-12-31", "files": [ "default_portfolio.csv" ], diff --git a/tests/config/full_params_2022Q4.json b/tests/config/full_params_2022Q4.json index c25ec83..661140b 100644 --- a/tests/config/full_params_2022Q4.json +++ b/tests/config/full_params_2022Q4.json @@ -4,7 +4,7 @@ "benchmarksURL": "https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2022Q4/2022Q4_20240529T002407Z", "parameters": { "portfolio": { - "holdingsDate": "2023-12-31", + "holdingsDate": "2022-12-31", "files": [ "default_portfolio.csv" ], @@ -35,7 +35,7 @@ "Aviation", "Cement" ], - "startYear": 2023, + "startYear": 2022, "timeHorizon": 5 }, "user": { diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json index a232958..6d42ad5 100644 --- a/tests/config/full_params_2023Q4.json +++ b/tests/config/full_params_2023Q4.json @@ -64,8 +64,8 @@ ], "portfolioAllocationMethod": "portfolio_weight", "scenarioGeography": "Global", - "scenarioOther": "GECO2022_1.5C", - "selectScenario": "WEO2022_NZE_2050", + "scenarioOther": "WEO2023_NZE_2050", + "selectScenario": "WEO2023_NZE_2050", "techRoadmapSectors": [ "Power", "Automotive", From 7e09a388aac9d2460dfaea74136ec420fa81e2a1 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 14:13:15 +0100 Subject: [PATCH 27/33] Clean gh-pages workflow --- .github/workflows/gh-pages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 64b3e9f..20387d4 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -1,3 +1,4 @@ +--- name: Deploy to GitHub Pages on: @@ -74,4 +75,3 @@ jobs: preview-branch: gh-pages umbrella-dir: pr-preview action: auto - dry-run: true From a426a64335d738a3f77be441f81c1293a74bc486 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 14:14:21 +0100 Subject: [PATCH 28/33] Clean test workflow file --- .github/workflows/test.yml | 52 -------------------------------------- 1 file changed, 52 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c5fe506..03902d6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -145,61 +145,9 @@ jobs: run: | ls -lR $DASHBOARD_OUTPUT_DIR - # - name: prepare artifact - # env: - # CONFIG_NAME: ${{ inputs.config-name }} - # DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }} - # run: | - # tar -cv -f "$CONFIG_NAME.tar.gz" -C "$DASHBOARD_OUTPUT_DIR" . - - - name: Upload built dashboard as artifact uses: actions/upload-artifact@v4 with: name: dashboard-build-${{ inputs.config-name }} path: ${{ steps.prepare.outputs.dashboard-output-dir }} if-no-files-found: error - - # - name: Prepare comment artifact - # id: prepare-artifact - # env: - # commit_time: ${{ github.event.pull_request.updated_at }} - # config_name: ${{ inputs.config-name }} - # full_image_name: ${{ inputs.full-image-name }} - # git_sha: ${{ github.event.pull_request.head.sha }} - # report_url: ${{ steps.export-outputs.outputs.report-url }} - # summary_url: ${{ steps.export-outputs.outputs.summary-url }} - # run: | - # mkdir -p /tmp/comment-json - # json_filename="$( \ - # echo "report-comment-json-merge-$full_image_name-$config_name" | \ - # tr '/:._' '-' \ - # ).json" - # echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" - # json_file="/tmp/comment-json/$json_filename" - # jq \ - # -n \ - # --arg commit_time "$commit_time" \ - # --arg config_name "$config_name" \ - # --arg git_sha "$git_sha" \ - # --arg report_url "$report_url" \ - # --arg summary_url "$summary_url" \ - # --arg full_image_name "$full_image_name" \ - # '{ - # "commit_time": $commit_time, - # "git_sha": $git_sha, - # "config": $config_name, - # "report": ("[Report](" + $report_url + ")"), - # "summary": (if (("" + $summary_url) == "") then null else "[Summary](" + $summary_url + ")" end), - # "image": ("`" + $full_image_name + "`") - # }' \ - # >> $json_file - # cat "$json_file" - - # - name: Upload comment JSON - # uses: actions/upload-artifact@v4 - # with: - # name: ${{ steps.prepare-artifact.outputs.json-filename }} - # path: /tmp/comment-json/* - # if-no-files-found: error - # retention-days: 1 From c0c6807815369dd341ea5713c072aa3c9c2f495e Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 14:20:11 +0100 Subject: [PATCH 29/33] Update and render Docs --- NAMESPACE | 1 + R/run_dashboard_workflow.R | 9 +++---- man/run_dashboard_workflow.Rd | 47 +++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 man/run_dashboard_workflow.Rd diff --git a/NAMESPACE b/NAMESPACE index 7eff832..97012a7 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,5 +1,6 @@ # Generated by roxygen2: do not edit by hand +export(run_dashboard_workflow) importFrom(dplyr,across) importFrom(dplyr,all_of) importFrom(dplyr,arrange) diff --git a/R/run_dashboard_workflow.R b/R/run_dashboard_workflow.R index 6843fa2..b5f686b 100644 --- a/R/run_dashboard_workflow.R +++ b/R/run_dashboard_workflow.R @@ -6,16 +6,13 @@ #' dashboard process. See JSON Schema for details. #' @param raw_params (`character`) Raw JSON string of parameters. #' @param run_analysis (`logical`) Run the analysis process. +#' @param run_dashboard (`logical`) Run the dashboard creation process. #' @param analysis_output_dir Directory containing the PACTA analysis results. -#' @param benchmarks_dir Directory containing the benchmark analysis results. +#' @param benchmarks_dir filepath: Directory containing the benchmark analysis +#' results. #' @param pacta_data_dir filepath: Directory with "pacta-data" #' @param portfolio_dir filepath: Directory with portfolio files -#' @param real_estate_dir Directory containing real estate data. #' @param dashboard_output_dir Directory where the dashboard will be -#' @param score_card_dir Directory containing score card data. -#' @param summary_output_dir Directory where the executive summary will be -#' @param survey_dir Directory containing survey data. -#' saved. #' saved. #' @return (invisible) `TRUE` if the workflow was successful. Primarily called #' for side effect of rendering files. diff --git a/man/run_dashboard_workflow.Rd b/man/run_dashboard_workflow.Rd new file mode 100644 index 0000000..fda7bf4 --- /dev/null +++ b/man/run_dashboard_workflow.Rd @@ -0,0 +1,47 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/run_dashboard_workflow.R +\name{run_dashboard_workflow} +\alias{run_dashboard_workflow} +\title{Run Dashboard workflow} +\usage{ +run_dashboard_workflow( + params, + raw_params, + run_analysis = TRUE, + run_dashboard = TRUE, + analysis_output_dir = Sys.getenv("ANALYSIS_OUTPUT_DIR"), + benchmarks_dir = Sys.getenv("BENCHMARKS_DIR"), + pacta_data_dir = Sys.getenv("PACTA_DATA_DIR"), + portfolio_dir = Sys.getenv("PORTFOLIO_DIR"), + dashboard_output_dir = Sys.getenv("DASHBOARD_OUTPUT_DIR") +) +} +\arguments{ +\item{params}{(\code{list}) A list of parameters to be used in the analysis and +dashboard process. See JSON Schema for details.} + +\item{raw_params}{(\code{character}) Raw JSON string of parameters.} + +\item{run_analysis}{(\code{logical}) Run the analysis process.} + +\item{run_dashboard}{(\code{logical}) Run the dashboard creation process.} + +\item{analysis_output_dir}{Directory containing the PACTA analysis results.} + +\item{benchmarks_dir}{filepath: Directory containing the benchmark analysis +results.} + +\item{pacta_data_dir}{filepath: Directory with "pacta-data"} + +\item{portfolio_dir}{filepath: Directory with portfolio files} + +\item{dashboard_output_dir}{Directory where the dashboard will be +saved.} +} +\value{ +(invisible) \code{TRUE} if the workflow was successful. Primarily called +for side effect of rendering files. +} +\description{ +Reun steps required to prepare a PACTA analysis and dashboard +} From 782ae741d5fa853df4e5b016d754c8c26e305074 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 14:44:44 +0100 Subject: [PATCH 30/33] Update Scenario references Co-authored-by: Jackson Hoffart --- inst/extdata/parameters/report_GENERAL_2023Q4.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/inst/extdata/parameters/report_GENERAL_2023Q4.json b/inst/extdata/parameters/report_GENERAL_2023Q4.json index d7ac08d..943285e 100644 --- a/inst/extdata/parameters/report_GENERAL_2023Q4.json +++ b/inst/extdata/parameters/report_GENERAL_2023Q4.json @@ -1,7 +1,7 @@ { "reporting": { - "scenarioOther": "GECO2022_1.5C", - "selectScenario": "WEO2022_NZE_2050" + "scenarioOther": "WEO2023_NZE_2050", + "selectScenario": "WEO2023_NZE_2050" }, "inherit": "report_GENERAL" } From 2904e0a7d5bd936e53bab6ae6cc7741d7d238e76 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 16:03:25 +0100 Subject: [PATCH 31/33] use porfolio schema from `workflow.pacta` rather than redfinining here --- NAMESPACE | 1 + R/prepare_schema_files.R | 54 +++++++++++++++++++ inst/extdata/schema/portfolio.json | 36 ------------- inst/extdata/scripts/prepare_dashboard_data.R | 15 +++--- man/prepare_schema_files.Rd | 19 +++++++ 5 files changed, 81 insertions(+), 44 deletions(-) create mode 100644 R/prepare_schema_files.R delete mode 100644 inst/extdata/schema/portfolio.json create mode 100644 man/prepare_schema_files.Rd diff --git a/NAMESPACE b/NAMESPACE index 97012a7..486fe37 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,5 +1,6 @@ # Generated by roxygen2: do not edit by hand +export(prepare_schema_files) export(run_dashboard_workflow) importFrom(dplyr,across) importFrom(dplyr,all_of) diff --git a/R/prepare_schema_files.R b/R/prepare_schema_files.R new file mode 100644 index 0000000..2b66774 --- /dev/null +++ b/R/prepare_schema_files.R @@ -0,0 +1,54 @@ +#' Copy parameter files to directory +#' +#' Copy schema files from dependency packages to a (temporary) directory. This +#' is required, as `jsonvalidate` requires referenced schema files to exist in +#' the same directory as the referencing schema. +#' +#' @param directory Directory to copy parameter files to. +#' @return directory +#' +#' @export +prepare_schema_files <- function(directory) { + log_debug("Preparing schema files.") + log_debug("Schema tempdir: ", schema_tempdir) + dashboard_schema_files <- list.files( + system.file( + "extdata", "schema", + package = "workflow.pacta.dashboard" + ), + full.names = TRUE + ) + dashboard_schema_copied <- file.copy( + from = dashboard_schema_files, + to = file.path( + directory, + basename(dashboard_schema_files) + ) + ) + + portfolio_schema_copied <- file.copy( + from = system.file( + "extdata", "schema", "portfolio.json", + package = "workflow.pacta" + ), + to = file.path( + directory, + "portfolio.json" + ) + ) + portfolio_schema_copied <- file.copy( + from = system.file( + "extdata", "schema", "portfolioParameters.json", + package = "workflow.pacta" + ), + to = file.path( + directory, + "portfolioParameters.json" + ) + ) + stopifnot( + dashboard_schema_copied, + portfolio_schema_copied + ) + return(directory) +} diff --git a/inst/extdata/schema/portfolio.json b/inst/extdata/schema/portfolio.json deleted file mode 100644 index cc70aaa..0000000 --- a/inst/extdata/schema/portfolio.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://raw.githubusercontent.com/RMI-PACTA/workflow.pacta/main/inst/extdata/schema/portfolio.json", - "title": "portfolio", - "description": "Definition for PACTA portfolio object", - "$comment": "Created by Alex Axthelm, aaxthelm@rmi.org", - "type": "object", - "properties": { - "files": { - "type": "array", - "description": "List of portfolio files to be analyzed.", - "items": { - "type": "string", - "description": "Path to the file." - } - }, - "holdingsDate": { - "type": "string", - "description": "Date of the holdings data.", - "format": "date", - "enum": [ - "2022-12-31", - "2023-12-31" - ] - }, - "name": { - "type": "string", - "description": "Name of the portfolio." - } - }, - "required": [ - "files", - "holdingsDate" - ], - "additionalProperties": false -} diff --git a/inst/extdata/scripts/prepare_dashboard_data.R b/inst/extdata/scripts/prepare_dashboard_data.R index 49aad2c..ceb6413 100644 --- a/inst/extdata/scripts/prepare_dashboard_data.R +++ b/inst/extdata/scripts/prepare_dashboard_data.R @@ -1,5 +1,10 @@ logger::log_threshold(Sys.getenv("LOG_LEVEL", "INFO")) +schema_tempdir <- tempdir() +workflow.pacta.dashboard:::prepare_schema_files( + directory = schema_tempdir +) + raw_params <- commandArgs(trailingOnly = TRUE) params <- pacta.workflow.utils::parse_raw_params( json = raw_params, @@ -13,14 +18,8 @@ params <- pacta.workflow.utils::parse_raw_params( package = "workflow.pacta" ) ), - schema_file = system.file( - "extdata", "schema", "reportingParameters.json", - package = "workflow.pacta.dashboard" - ), - raw_schema_file = system.file( - "extdata", "schema", "rawParameters.json", - package = "workflow.pacta.dashboard" - ), + schema_file = file.path(schema_tempdir, "reportingParameters.json"), + raw_schema_file = file.path(schema_tempdir, "rawParameters.json"), force_array = c("portfolio", "files") ) diff --git a/man/prepare_schema_files.Rd b/man/prepare_schema_files.Rd new file mode 100644 index 0000000..6dc2cb4 --- /dev/null +++ b/man/prepare_schema_files.Rd @@ -0,0 +1,19 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/prepare_schema_files.R +\name{prepare_schema_files} +\alias{prepare_schema_files} +\title{Copy parameter files to directory} +\usage{ +prepare_schema_files(directory) +} +\arguments{ +\item{directory}{Directory to copy parameter files to.} +} +\value{ +directory +} +\description{ +Copy schema files from dependency packages to a (temporary) directory. This +is required, as \code{jsonvalidate} requires referenced schema files to exist in +the same directory as the referencing schema. +} From eb8d23ee9084dba9847848a6a87ee36f217dae7e Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 16:07:21 +0100 Subject: [PATCH 32/33] Use local variable name --- R/prepare_schema_files.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/prepare_schema_files.R b/R/prepare_schema_files.R index 2b66774..40d9749 100644 --- a/R/prepare_schema_files.R +++ b/R/prepare_schema_files.R @@ -10,7 +10,7 @@ #' @export prepare_schema_files <- function(directory) { log_debug("Preparing schema files.") - log_debug("Schema tempdir: ", schema_tempdir) + log_debug("Schema tempdir: ", directory) dashboard_schema_files <- list.files( system.file( "extdata", "schema", From 06a57c99e82c1fa9f5794fa4834899f7281bdb1e Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 13 Dec 2024 16:36:13 +0100 Subject: [PATCH 33/33] Update and render docs --- R/run_dashboard_workflow.R | 2 +- man/run_dashboard_workflow.Rd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/run_dashboard_workflow.R b/R/run_dashboard_workflow.R index b5f686b..0a71c2f 100644 --- a/R/run_dashboard_workflow.R +++ b/R/run_dashboard_workflow.R @@ -1,6 +1,6 @@ #' Run Dashboard workflow #' -#' Reun steps required to prepare a PACTA analysis and dashboard +#' Run steps required to prepare a PACTA analysis and dashboard #' #' @param params (`list`) A list of parameters to be used in the analysis and #' dashboard process. See JSON Schema for details. diff --git a/man/run_dashboard_workflow.Rd b/man/run_dashboard_workflow.Rd index fda7bf4..ffda951 100644 --- a/man/run_dashboard_workflow.Rd +++ b/man/run_dashboard_workflow.Rd @@ -43,5 +43,5 @@ saved.} for side effect of rendering files. } \description{ -Reun steps required to prepare a PACTA analysis and dashboard +Run steps required to prepare a PACTA analysis and dashboard }