From 5696c6527accd009c5088ddeded9577317adb734 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 13:16:43 +0200 Subject: [PATCH 001/132] Add docker action from RMI-PACTA/actions --- .github/workflows/docker.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..e755ec5 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,31 @@ +--- +on: + pull_request: + push: + branches: [main] + schedule: + - cron: '0 0 * * 1,2,3,4,5' + workflow_dispatch: + +name: docker + +jobs: + docker: + name: Docker actions + uses: RMI-PACTA/actions/.github/workflows/docker.yml@main + + # test: + # name: Test + # uses: ./.github/workflows/test.yml + # needs: [docker] + # secrets: inherit + # strategy: + # fail-fast: false + # matrix: + # config-name: + # - default_2022Q4 + # - default_2023Q4 + # - full_params_2023Q4 + # with: + # full-image-name: ${{ needs.docker.outputs.full-image-name }} + # config-name: ${{ matrix.config-name }} From d35a897af6f6def58e7ec81fb819a9903fd9de06 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 13:18:05 +0200 Subject: [PATCH 002/132] Remvoe old actions --- .../workflows/build-Docker-image-nightly.yml | 12 ---- .../build-Docker-image-on-push-to-main.yml | 12 ---- .../build-Docker-image-on-push-to-pr.yml | 37 ---------- .../workflows/build-and-push-Docker-image.yml | 67 ------------------- .github/workflows/check-R-sysdeps.yml | 32 --------- .github/workflows/run-hadolint.yml | 11 --- 6 files changed, 171 deletions(-) delete mode 100644 .github/workflows/build-Docker-image-nightly.yml delete mode 100644 .github/workflows/build-Docker-image-on-push-to-main.yml delete mode 100644 .github/workflows/build-Docker-image-on-push-to-pr.yml delete mode 100644 .github/workflows/build-and-push-Docker-image.yml delete mode 100644 .github/workflows/check-R-sysdeps.yml delete mode 100644 .github/workflows/run-hadolint.yml diff --git a/.github/workflows/build-Docker-image-nightly.yml b/.github/workflows/build-Docker-image-nightly.yml deleted file mode 100644 index 546b98b..0000000 --- a/.github/workflows/build-Docker-image-nightly.yml +++ /dev/null @@ -1,12 +0,0 @@ -on: - schedule: - - cron: '0 0 * * 1,2,3,4,5' - -jobs: - build_docker_image: - name: "Call build and push action" - uses: ./.github/workflows/build-and-push-Docker-image.yml - secrets: inherit - with: - image-name: workflow.pacta.report - image-tag: nightly diff --git a/.github/workflows/build-Docker-image-on-push-to-main.yml b/.github/workflows/build-Docker-image-on-push-to-main.yml deleted file mode 100644 index a442b85..0000000 --- a/.github/workflows/build-Docker-image-on-push-to-main.yml +++ /dev/null @@ -1,12 +0,0 @@ -on: - push: - branches: [main] - -jobs: - build_docker_image: - name: "Call build and push action" - uses: ./.github/workflows/build-and-push-Docker-image.yml - secrets: inherit - with: - image-name: workflow.pacta.report - image-tag: main diff --git a/.github/workflows/build-Docker-image-on-push-to-pr.yml b/.github/workflows/build-Docker-image-on-push-to-pr.yml deleted file mode 100644 index cf4b0f9..0000000 --- a/.github/workflows/build-Docker-image-on-push-to-pr.yml +++ /dev/null @@ -1,37 +0,0 @@ -on: - pull_request: - -jobs: - build_docker_image: - name: "Call build and push action" - uses: ./.github/workflows/build-and-push-Docker-image.yml - secrets: inherit - with: - image-name: workflow.pacta.report - image-tag: pr${{ github.event.pull_request.number }} - - add_comment: - needs: build_docker_image - runs-on: ubuntu-latest - steps: - - name: Find Comment - # https://github.com/peter-evans/find-comment - uses: peter-evans/find-comment@v2 - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: Docker image from this PR - - - name: Create or update comment - # https://github.com/peter-evans/create-or-update-comment - uses: peter-evans/create-or-update-comment@v3 - with: - comment-id: ${{ steps.fc.outputs.comment-id }} - issue-number: ${{ github.event.pull_request.number }} - body: | - Docker image from this PR (${{ github.event.pull_request.head.sha }}) created - ``` - docker pull ${{ needs.build_docker_image.outputs.full-image-name }} - ``` - edit-mode: replace diff --git a/.github/workflows/build-and-push-Docker-image.yml b/.github/workflows/build-and-push-Docker-image.yml deleted file mode 100644 index b6d8e1e..0000000 --- a/.github/workflows/build-and-push-Docker-image.yml +++ /dev/null @@ -1,67 +0,0 @@ ---- -name: Build and push docker image - -on: - workflow_call: - inputs: - image-name: - required: true - type: string - image-tag: - required: true - type: string - outputs: - full-image-name: - description: "Full pushed image name including host/registry, name, and tag" - value: ${{ jobs.docker.outputs.full-image-name }} - -jobs: - docker: - runs-on: ubuntu-latest - permissions: - packages: write - contents: read - timeout-minutes: 25 - outputs: - full-image-name: ${{ steps.image-name.outputs.full-image-name }} - - steps: - - - name: Define image name - id: image-name - run: | - full_image_name="ghcr.io/${{ github.repository_owner }}/${{ inputs.image-name }}:${{ inputs.image-tag }}" - full_image_name=$(echo $full_image_name | tr '[A-Z]' '[a-z]') - echo "full-image-name=$full_image_name" >> "$GITHUB_OUTPUT" - echo "$full_image_name" > full-image-name - - - uses: actions/upload-artifact@v3 - with: - name: full-image-name - path: . - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push - uses: docker/build-push-action@v5 - with: - push: true - tags: ${{ steps.image-name.outputs.full-image-name }} - cache-from: type=gha - cache-to: type=gha,mode=min - no-cache-filters: install-pacta - - check-system-dependencies: - name: "Check System Dependencies" - needs: docker - uses: ./.github/workflows/check-R-sysdeps.yml - with: - image: ${{ needs.docker.outputs.full-image-name }} \ No newline at end of file diff --git a/.github/workflows/check-R-sysdeps.yml b/.github/workflows/check-R-sysdeps.yml deleted file mode 100644 index 3a1c08b..0000000 --- a/.github/workflows/check-R-sysdeps.yml +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: Check R system dependencies - -on: - workflow_call: - inputs: - image: - required: true - type: string - -jobs: - - check-system-dependencies: - runs-on: ubuntu-latest - steps: - - name: 'Pull image' - run: | - echo ${{ inputs.image }} - docker pull ${{ inputs.image }} - - name: 'Run pak::sysreqs_check_installed()' - run: | - - docker run \ - --rm \ - --entrypoint "/bin/sh" \ - ${{ inputs.image }} \ - -c "Rscript -e ' - x <- pak::sysreqs_check_installed() - print(x) - is_installed <- as.data.frame(x)[[\"installed\"]] - stopifnot(all(is_installed)) - '" diff --git a/.github/workflows/run-hadolint.yml b/.github/workflows/run-hadolint.yml deleted file mode 100644 index 0f07812..0000000 --- a/.github/workflows/run-hadolint.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -on: [push, pull_request] - -jobs: - hadolint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: hadolint/hadolint-action@v3.1.0 - with: - dockerfile: Dockerfile From c55c309853a75ca777c069921a26981417d24707 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 13:23:38 +0200 Subject: [PATCH 003/132] Update Docker labels --- Dockerfile | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 062a4a5..d7c6e65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,13 @@ -# using rocker r-vers as a base with R 4.3.1 -# https://hub.docker.com/r/rocker/r-ver -# https://rocker-project.org/images/versioned/r-ver.html -# -# sets CRAN repo to use Posit Package Manager to freeze R package versions to -# those available on 2023-10-30 -# https://packagemanager.posit.co/client/#/repos/2/overview -# https://packagemanager.posit.co/cran/__linux__/jammy/2023-10-30 - -# set proper base image -ARG R_VERS="4.3.1" FROM ghcr.io/rmi-pacta/workflow.pacta:main AS base # set Docker image labels LABEL org.opencontainers.image.source=https://github.com/RMI-PACTA/workflow.pacta.report -LABEL org.opencontainers.image.description="Docker image to run PACTA" +LABEL org.opencontainers.image.description="Docker image to create PACTA reports and executive summaries" LABEL org.opencontainers.image.licenses=MIT -LABEL org.opencontainers.image.title="" -LABEL org.opencontainers.image.revision="" -LABEL org.opencontainers.image.version="" -LABEL org.opencontainers.image.vendor="" -LABEL org.opencontainers.image.base.name="" -LABEL org.opencontainers.image.ref.name="" -LABEL org.opencontainers.image.authors="" +LABEL org.opencontainers.image.title="workflow.pacta.report" +LABEL org.opencontainers.image.vendor="RMI" +LABEL org.opencontainers.image.base.name="ghcr.io/rmi-pacta/workflow.pacta:main" +LABEL org.opencontainers.image.authors="Alex Axthelm" # set apt-get to noninteractive mode ARG DEBIAN_FRONTEND="noninteractive" From bf1935fc8b5b51656d8d1bf100c7ad2fbeca2773 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 13:24:41 +0200 Subject: [PATCH 004/132] Clean up sysdep installation --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index d7c6e65..316e45c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,13 +9,12 @@ LABEL org.opencontainers.image.vendor="RMI" LABEL org.opencontainers.image.base.name="ghcr.io/rmi-pacta/workflow.pacta:main" LABEL org.opencontainers.image.authors="Alex Axthelm" -# set apt-get to noninteractive mode -ARG DEBIAN_FRONTEND="noninteractive" -ARG DEBCONF_NOWARNINGS="yes" +USER root # install system dependencies RUN apt-get update \ - && apt-get install -y --no-install-recommends \ + && DEBIAN_FRONTEND="noninteractive" \ + apt-get install -y --no-install-recommends \ libpng-dev=1.6.* \ libxt6=1:1.2.* \ pandoc=2.9.* \ From f8673d88075ab28b0e75ecf88f1ededf6cea7531 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 13:29:11 +0200 Subject: [PATCH 005/132] Simplify dependency installation; add non-root user --- Dockerfile | 49 +++++++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 316e45c..b911083 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,41 +21,30 @@ RUN apt-get update \ && chmod -R a+rwX /root \ && rm -rf /var/lib/apt/lists/* -# set frozen CRAN repo -ARG CRAN_REPO="https://packagemanager.posit.co/cran/__linux__/jammy/2024-03-05" -RUN echo "options(repos = c(CRAN = '$CRAN_REPO'), pkg.sysreqs = FALSE)" >> "${R_HOME}/etc/Rprofile.site" \ - # install packages for dependency resolution and installation - && Rscript -e "install.packages('pak'); pak::pkg_install('renv')" - -FROM base AS install-pacta - # copy in everything from this repo -COPY DESCRIPTION /DESCRIPTION +COPY DESCRIPTION /workflow.pacta.report/DESCRIPTION + +# Rprofile, including CRAN-like repos are inhertied from base image +# install pak, find dependencises from DESCRIPTION, and install them. +RUN Rscript -e "\ + install.packages('pak'); \ + deps <- pak::local_deps(root = '/workflow.pacta.report'); \ + pkg_deps <- deps[!deps[['direct']], 'ref']; \ + print(pkg_deps); \ + pak::pak(pkg_deps); \ + " -# PACTA R package tags -ARG report_tag="/tree/main" -ARG summary_tag="/tree/main" -ARG utils_tag="/tree/main" +FROM base AS install-pacta -ARG report_url="https://github.com/rmi-pacta/pacta.portfolio.report" -ARG summary_url="https://github.com/rmi-pacta/pacta.executive.summary" -ARG utils_url="https://github.com/rmi-pacta/pacta.portfolio.utils" +COPY . /workflow.pacta.report/ -# install R package dependencies -RUN Rscript -e "\ - gh_pkgs <- \ - c( \ - paste0('$report_url', '$report_tag'), \ - paste0('$summary_url', '$summary_tag'), \ - paste0('$utils_url', '$utils_tag') \ - ); \ - workflow_pkgs <- renv::dependencies('DESCRIPTION')[['Package']]; \ - workflow_pkgs <- grep('^pacta[.]', workflow_pkgs, value = TRUE, invert = TRUE); \ - pak::pak(c(gh_pkgs, workflow_pkgs)); \ - " - -COPY . / +RUN Rscript -e "pak::local_install(root = '/workflow.pacta.report')" # set default run behavior ENTRYPOINT ["/run-pacta.sh"] CMD ["input_dir/default_config.json"] + +# Create and use non-root user +RUN useradd -m -s /bin/bash workflow-pacta-report +USER workflow-pacta-report +WORKDIR /home/workflow-pacta-report From 247aab987a16432660972da897fe5a3a48ffef00 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 13:36:55 +0200 Subject: [PATCH 006/132] Move user creation to just after installing sysdeps --- Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index b911083..0365f32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,11 @@ RUN apt-get update \ && chmod -R a+rwX /root \ && rm -rf /var/lib/apt/lists/* +# Create and use non-root user +RUN useradd -m -s /bin/bash workflow-pacta-report +USER workflow-pacta-report +WORKDIR /home/workflow-pacta-report + # copy in everything from this repo COPY DESCRIPTION /workflow.pacta.report/DESCRIPTION @@ -43,8 +48,3 @@ RUN Rscript -e "pak::local_install(root = '/workflow.pacta.report')" # set default run behavior ENTRYPOINT ["/run-pacta.sh"] CMD ["input_dir/default_config.json"] - -# Create and use non-root user -RUN useradd -m -s /bin/bash workflow-pacta-report -USER workflow-pacta-report -WORKDIR /home/workflow-pacta-report From c010df2470ab1d91cea4506d079901c1c86ea8ae Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 13:45:41 +0200 Subject: [PATCH 007/132] Add DESCRIPTION and other R package skeleton --- .Rbuildignore | 1 + DESCRIPTION | 46 +++++++++++++++++++++++++++++++++++----------- LICENSE | 23 ++--------------------- LICENSE.md | 21 +++++++++++++++++++++ NAMESPACE | 2 ++ 5 files changed, 61 insertions(+), 32 deletions(-) create mode 100644 .Rbuildignore create mode 100644 LICENSE.md create mode 100644 NAMESPACE diff --git a/.Rbuildignore b/.Rbuildignore new file mode 100644 index 0000000..5163d0b --- /dev/null +++ b/.Rbuildignore @@ -0,0 +1 @@ +^LICENSE\.md$ diff --git a/DESCRIPTION b/DESCRIPTION index f108a0e..f262ad3 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,11 +1,35 @@ -Type: PACTA -Description: Run PACTA. -Depends: - dplyr, - fs, - jsonlite, - logger, - pacta.executive.summary, - pacta.portfolio.report, - pacta.portfolio.utils, - readr +Package: workflow.pacta.report +Title: Reporting functionality for PACTA +Version: 0.0.0.9000 +Authors@R: + c(person(given = "Alex", + family = "Axthelm", + role = c("aut", "cre", "ctr"), + email = "aaxthelm@rmi.org", + comment = c(ORCID = "0000-0001-8579-8565")), + person(given = "Jackson", + family = "Hoffart", + email = "jackson.hoffart@gmail.com", + role = c("aut", "ctr"), + comment = c(ORCID = "0000-0002-8600-5042")), + person(given = "RMI", + role = c("cph", "fnd"), + email = "PACTA4investors@rmi.org")) +Description: Create Interactive Reports and Executive Summaries for PACTA +License: MIT + file LICENSE +Encoding: UTF-8 +Roxygen: list(markdown = TRUE) +RoxygenNote: 7.3.1 +Imports: + dplyr, + fs, + jsonlite, + logger, + pacta.executive.summary, + pacta.portfolio.report, + pacta.portfolio.utils, + readr +Remotes: + RMI-PACTA/pacta.executive.summary, + RMI-PACTA/pacta.portfolio.report, + RMI-PACTA/pacta.portfolio.utils diff --git a/LICENSE b/LICENSE index 844b42c..7ec6c95 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,2 @@ -MIT License - -Copyright (c) 2023 RMI - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +YEAR: 2024 +COPYRIGHT HOLDER: RMI diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..c3f0481 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,21 @@ +# MIT License + +Copyright (c) 2024 RMI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NAMESPACE b/NAMESPACE new file mode 100644 index 0000000..6ae9268 --- /dev/null +++ b/NAMESPACE @@ -0,0 +1,2 @@ +# Generated by roxygen2: do not edit by hand + From c6bab7819f523144857044aa441b5ba7253ee79a Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 14:14:13 +0200 Subject: [PATCH 008/132] Add user to `staff`, allowing R package install --- Dockerfile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0365f32..7153159 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,12 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # Create and use non-root user -RUN useradd -m -s /bin/bash workflow-pacta-report +# -m creates a home directory, +# -G adds user to staff group allowing R package installation. +RUN useradd \ + -m \ + -G staff \ + workflow-pacta-report USER workflow-pacta-report WORKDIR /home/workflow-pacta-report From b5005f59516f72f1c01643f1129c16b5dc9fdab2 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 15:04:56 +0200 Subject: [PATCH 009/132] Use base image with newer CRAN pin date --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7153159..a52845b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/rmi-pacta/workflow.pacta:main AS base +FROM ghcr.io/rmi-pacta/workflow.pacta:pr-71 AS base # set Docker image labels LABEL org.opencontainers.image.source=https://github.com/RMI-PACTA/workflow.pacta.report From fb28331d1f8240800cf4a8d8b7481697c6a2d689 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 15:06:36 +0200 Subject: [PATCH 010/132] Add `docker-compose.yml` --- docker-compose.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..d82b0aa --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,27 @@ +--- +services: + workflow.pacta.report: + build: . + stdin_open: true + tty: true + # entrypoint: ["R", "--args"] + # command: '{\"portfolio_files\": \"default_portfolio.csv\", \"inherit\": \"GENERAL_2023Q4\"}' + entrypoint: "bash" + environment: + LOG_LEVEL: TRACE + PACTA_DATA_DIR: "/mnt/pacta-data" + OUTPUT_DIR: "/mnt/output_dir" + PORTFOLIO_DIR: "/mnt/portfolios" + # volumes: + # - type: bind + # source: ${data_dir:-./pacta-data} + # target: /mnt/pacta-data + # read_only: true + # - type: bind + # source: ${output_dir:-./output_dir} + # target: /mnt/output_dir + # read_only: false + # - type: bind + # source: ${input_dir:-./portfolios} + # target: /mnt/portfolios + # read_only: true From b5e0c27113843abf367f7f448e1f1fe66e70db08 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 18:12:34 +0200 Subject: [PATCH 011/132] Run `pak` commands as `root` Package installation is not working, see r-lib/pak, Issue #638 --- Dockerfile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a52845b..5c625a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,9 +9,8 @@ LABEL org.opencontainers.image.vendor="RMI" LABEL org.opencontainers.image.base.name="ghcr.io/rmi-pacta/workflow.pacta:main" LABEL org.opencontainers.image.authors="Alex Axthelm" -USER root - # install system dependencies +USER root RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" \ apt-get install -y --no-install-recommends \ @@ -34,6 +33,7 @@ WORKDIR /home/workflow-pacta-report # copy in everything from this repo COPY DESCRIPTION /workflow.pacta.report/DESCRIPTION +USER root #TODO: remove this line # Rprofile, including CRAN-like repos are inhertied from base image # install pak, find dependencises from DESCRIPTION, and install them. RUN Rscript -e "\ @@ -43,12 +43,15 @@ RUN Rscript -e "\ print(pkg_deps); \ pak::pak(pkg_deps); \ " +USER workflow-pacta-report FROM base AS install-pacta COPY . /workflow.pacta.report/ +USER root #TODO: remove this line RUN Rscript -e "pak::local_install(root = '/workflow.pacta.report')" +USER workflow-pacta-report # set default run behavior ENTRYPOINT ["/run-pacta.sh"] From ce149f373fc39fb787eb74f30d577fc8e3767419 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 18:15:34 +0200 Subject: [PATCH 012/132] Move non-root user to after package installation --- Dockerfile | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5c625a7..8e31728 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,20 +20,9 @@ RUN apt-get update \ && chmod -R a+rwX /root \ && rm -rf /var/lib/apt/lists/* -# Create and use non-root user -# -m creates a home directory, -# -G adds user to staff group allowing R package installation. -RUN useradd \ - -m \ - -G staff \ - workflow-pacta-report -USER workflow-pacta-report -WORKDIR /home/workflow-pacta-report - # copy in everything from this repo COPY DESCRIPTION /workflow.pacta.report/DESCRIPTION -USER root #TODO: remove this line # Rprofile, including CRAN-like repos are inhertied from base image # install pak, find dependencises from DESCRIPTION, and install them. RUN Rscript -e "\ @@ -43,15 +32,22 @@ RUN Rscript -e "\ print(pkg_deps); \ pak::pak(pkg_deps); \ " + +# Create and use non-root user +# -m creates a home directory, +# -G adds user to staff group allowing R package installation. +RUN useradd \ + -m \ + -G staff \ + workflow-pacta-report USER workflow-pacta-report +WORKDIR /home/workflow-pacta-report FROM base AS install-pacta COPY . /workflow.pacta.report/ -USER root #TODO: remove this line RUN Rscript -e "pak::local_install(root = '/workflow.pacta.report')" -USER workflow-pacta-report # set default run behavior ENTRYPOINT ["/run-pacta.sh"] From d8c5e53b2ca4b2b6f1615c73cc61193d256f8afd Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 18:21:53 +0200 Subject: [PATCH 013/132] toss it, just start fresh, and don't inherit --- Dockerfile | 46 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8e31728..5851402 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/rmi-pacta/workflow.pacta:pr-71 AS base +FROM docker.io/rocker/r-ver:4.3.1 AS base # set Docker image labels LABEL org.opencontainers.image.source=https://github.com/RMI-PACTA/workflow.pacta.report @@ -20,6 +20,40 @@ RUN apt-get update \ && chmod -R a+rwX /root \ && rm -rf /var/lib/apt/lists/* +# set frozen CRAN repo and RProfile.site +# This block makes use of the builtin ARG $TARGETPLATFORM (See: +# https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ +# ) to pick the correct CRAN-like repo, which will let us target binaries fo +# supported platforms +ARG TARGETPLATFORM +RUN PACKAGE_PIN_DATE="2023-10-30" && \ + echo "TARGETPLATFORM: $TARGETPLATFORM" && \ + if [ "$TARGETPLATFORM" = "linux/amd64" ] && grep -q -e "Jammy Jellyfish" "/etc/os-release" ; then \ + CRAN_LIKE_URL="https://packagemanager.posit.co/cran/__linux__/jammy/$PACKAGE_PIN_DATE"; \ + else \ + CRAN_LIKE_URL="https://packagemanager.posit.co/cran/$PACKAGE_PIN_DATE"; \ + fi && \ + echo "CRAN_LIKE_URL: $CRAN_LIKE_URL" && \ + printf "options(\n \ + repos = c(CRAN = '%s'),\n \ + pak.no_extra_messages = TRUE,\n \ + pkg.sysreqs = FALSE,\n \ + pkg.sysreqs_db_update = FALSE,\n \ + pkg.sysreqs_update = FALSE\n \ + )\n" \ + "$CRAN_LIKE_URL" \ + > "${R_HOME}/etc/Rprofile.site" + +# Create and use non-root user +# -m creates a home directory, +# -G adds user to staff group allowing R package installation. +RUN useradd \ + -m \ + -G staff \ + workflow-pacta-report +USER workflow-pacta-report +WORKDIR /home/workflow-pacta-report + # copy in everything from this repo COPY DESCRIPTION /workflow.pacta.report/DESCRIPTION @@ -33,16 +67,6 @@ RUN Rscript -e "\ pak::pak(pkg_deps); \ " -# Create and use non-root user -# -m creates a home directory, -# -G adds user to staff group allowing R package installation. -RUN useradd \ - -m \ - -G staff \ - workflow-pacta-report -USER workflow-pacta-report -WORKDIR /home/workflow-pacta-report - FROM base AS install-pacta COPY . /workflow.pacta.report/ From 8a5951d8040bbfcf8ee64aca3f8961a01ab1f6ba Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 28 May 2024 18:26:02 +0200 Subject: [PATCH 014/132] Also install pkgdepends --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5851402..5b8ea2f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,7 +60,7 @@ COPY DESCRIPTION /workflow.pacta.report/DESCRIPTION # Rprofile, including CRAN-like repos are inhertied from base image # install pak, find dependencises from DESCRIPTION, and install them. RUN Rscript -e "\ - install.packages('pak'); \ + install.packages(c('pak', 'pkgdepends')); \ deps <- pak::local_deps(root = '/workflow.pacta.report'); \ pkg_deps <- deps[!deps[['direct']], 'ref']; \ print(pkg_deps); \ From 2f870e34dcf84a21145e59a7419a338c67988249 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 15:12:46 +0200 Subject: [PATCH 015/132] Move `pak` installation into setup --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b8ea2f..12a45f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,7 +42,8 @@ RUN PACKAGE_PIN_DATE="2023-10-30" && \ pkg.sysreqs_update = FALSE\n \ )\n" \ "$CRAN_LIKE_URL" \ - > "${R_HOME}/etc/Rprofile.site" + > "${R_HOME}/etc/Rprofile.site" \ + && Rscript -e "install.packages('pak', repos = sprintf('https://r-lib.github.io/p/pak/stable/%s/%s/%s', .Platform[['pkgType']], R.Version()[['os']], R.Version()[['arch']]))" # Create and use non-root user # -m creates a home directory, @@ -60,7 +61,6 @@ COPY DESCRIPTION /workflow.pacta.report/DESCRIPTION # Rprofile, including CRAN-like repos are inhertied from base image # install pak, find dependencises from DESCRIPTION, and install them. RUN Rscript -e "\ - install.packages(c('pak', 'pkgdepends')); \ deps <- pak::local_deps(root = '/workflow.pacta.report'); \ pkg_deps <- deps[!deps[['direct']], 'ref']; \ print(pkg_deps); \ From 4c75a34f3ae96e2e7b0eae4150933bfb3f8f7812 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 15:15:32 +0200 Subject: [PATCH 016/132] Update pin date --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 12a45f0..11ba537 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ RUN apt-get update \ # ) to pick the correct CRAN-like repo, which will let us target binaries fo # supported platforms ARG TARGETPLATFORM -RUN PACKAGE_PIN_DATE="2023-10-30" && \ +RUN PACKAGE_PIN_DATE="2024-04-05" && \ echo "TARGETPLATFORM: $TARGETPLATFORM" && \ if [ "$TARGETPLATFORM" = "linux/amd64" ] && grep -q -e "Jammy Jellyfish" "/etc/os-release" ; then \ CRAN_LIKE_URL="https://packagemanager.posit.co/cran/__linux__/jammy/$PACKAGE_PIN_DATE"; \ From 68260158a60b8672b051c556a1edd43031aa32a4 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 15:21:13 +0200 Subject: [PATCH 017/132] Use `pka::local_install_deps()` --- Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 11ba537..c009225 100644 --- a/Dockerfile +++ b/Dockerfile @@ -60,12 +60,7 @@ COPY DESCRIPTION /workflow.pacta.report/DESCRIPTION # Rprofile, including CRAN-like repos are inhertied from base image # install pak, find dependencises from DESCRIPTION, and install them. -RUN Rscript -e "\ - deps <- pak::local_deps(root = '/workflow.pacta.report'); \ - pkg_deps <- deps[!deps[['direct']], 'ref']; \ - print(pkg_deps); \ - pak::pak(pkg_deps); \ - " +RUN Rscript -e "pak::local_install_deps(root = '/workflow.pacta.report')" FROM base AS install-pacta From 98da6847cd3d487f9d4c01ed1e26be014f1e8908 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 15:28:04 +0200 Subject: [PATCH 018/132] Add `libicu-dev` sysdep --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index c009225..93a10bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ USER root RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" \ apt-get install -y --no-install-recommends \ + libicu-dev=70.* \ libpng-dev=1.6.* \ libxt6=1:1.2.* \ pandoc=2.9.* \ From 4280d5f3b2cbaeb745c3f16798ba047d7dc6925f Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 17:15:42 +0200 Subject: [PATCH 019/132] Reflect new directory structure --- Dockerfile | 2 +- docker-compose.yml | 6 +++--- run-pacta.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 93a10bc..3d6c3db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,5 +70,5 @@ COPY . /workflow.pacta.report/ RUN Rscript -e "pak::local_install(root = '/workflow.pacta.report')" # set default run behavior -ENTRYPOINT ["/run-pacta.sh"] +ENTRYPOINT ["/workflow.pacta.report/run-pacta.sh"] CMD ["input_dir/default_config.json"] diff --git a/docker-compose.yml b/docker-compose.yml index d82b0aa..f9f6e55 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,11 +2,11 @@ services: workflow.pacta.report: build: . - stdin_open: true - tty: true + # stdin_open: true + # tty: true # entrypoint: ["R", "--args"] # command: '{\"portfolio_files\": \"default_portfolio.csv\", \"inherit\": \"GENERAL_2023Q4\"}' - entrypoint: "bash" + # entrypoint: "bash" environment: LOG_LEVEL: TRACE PACTA_DATA_DIR: "/mnt/pacta-data" diff --git a/run-pacta.sh b/run-pacta.sh index d24b05a..9b77174 100755 --- a/run-pacta.sh +++ b/run-pacta.sh @@ -3,4 +3,4 @@ # Set permissions so that new files can be deleted/overwritten outside docker umask 000 -Rscript --vanilla pacta_03.R "${1}" \ +Rscript --vanilla /workflow.pacta.report/pacta_03.R "${1}" \ From 0e8774a39c576a4359f283f3fc136706648870ad Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 20:50:18 +0200 Subject: [PATCH 020/132] Working build --- Dockerfile | 1 - docker-compose.yml | 27 ++++++++++++++------------- input_dir/default_config.json | 15 +++++++-------- pacta_03.R | 3 +-- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3d6c3db..386c4fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,4 +71,3 @@ RUN Rscript -e "pak::local_install(root = '/workflow.pacta.report')" # set default run behavior ENTRYPOINT ["/workflow.pacta.report/run-pacta.sh"] -CMD ["input_dir/default_config.json"] diff --git a/docker-compose.yml b/docker-compose.yml index f9f6e55..f2c4e9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,21 +7,22 @@ services: # entrypoint: ["R", "--args"] # command: '{\"portfolio_files\": \"default_portfolio.csv\", \"inherit\": \"GENERAL_2023Q4\"}' # entrypoint: "bash" + # command: "/workflow.pacta.report/input_dir/default_config.json" environment: LOG_LEVEL: TRACE PACTA_DATA_DIR: "/mnt/pacta-data" OUTPUT_DIR: "/mnt/output_dir" PORTFOLIO_DIR: "/mnt/portfolios" - # volumes: - # - type: bind - # source: ${data_dir:-./pacta-data} - # target: /mnt/pacta-data - # read_only: true - # - type: bind - # source: ${output_dir:-./output_dir} - # target: /mnt/output_dir - # read_only: false - # - type: bind - # source: ${input_dir:-./portfolios} - # target: /mnt/portfolios - # read_only: true + volumes: + - type: bind + source: ${data_dir:-./pacta-data} + target: /mnt/pacta-data/ + read_only: true + - type: bind + source: ${output_dir:-./output_dir} + target: /mnt/output_dir + read_only: false + # - type: bind + # source: ${input_dir:-./portfolios} + # target: /mnt/portfolios + # read_only: true diff --git a/input_dir/default_config.json b/input_dir/default_config.json index 1112fd7..642675e 100644 --- a/input_dir/default_config.json +++ b/input_dir/default_config.json @@ -1,18 +1,17 @@ { - "data_dir": "../pacta-data/2022Q4", - "portfolio_path": "input_dir/default_portfolio.csv", - "output_dir": "output_dir", - "start_year": 2022, + "data_dir": "/mnt/pacta-data/", + "portfolio_path": "/mnt/input_dir/default_portfolio.csv", + "output_dir": "/mnt/output_dir", + "start_year": 2023, "equity_market_list": [ "GlobalMarket", "DevelopedMarket", "EmergingMarket" ], "scenario_sources_list": [ - "GECO2022", - "IPR2021", - "ISF2021", - "WEO2022" + "GECO2023", + "ISF2023", + "WEO2023" ], "scenario_geographies_list": [ "Global", diff --git a/pacta_03.R b/pacta_03.R index 14a9c59..ae9399c 100644 --- a/pacta_03.R +++ b/pacta_03.R @@ -20,7 +20,7 @@ logger::log_trace("Determining configuration file path") cfg_path <- commandArgs(trailingOnly = TRUE) if (length(cfg_path) == 0 || cfg_path == "") { logger::log_warn("No configuration file specified, using default") - cfg_path <- "input_dir/default_config.json" + cfg_path <- "/workflow.pacta.report/input_dir/default_config.json" } logger::log_debug("Loading configuration from file: \"{cfg_path}\".") cfg <- fromJSON(cfg_path) @@ -176,7 +176,6 @@ indices_equity_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_equ logger::log_debug("Loading index bonds portfolio results.") indices_bonds_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_bonds_results_portfolio.rds")) - # create interactive report ---------------------------------------------------- logger::log_debug("Preparing to create interactive report.") From aa5f323cc074cc6934cea652ccf362805edbe2cf Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 21:12:36 +0200 Subject: [PATCH 021/132] add script, lightly modified from `workflow.pacta` --- .github/workflows/docker.yml | 30 ++++---- .github/workflows/test.yml | 133 +++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e755ec5..1b9a4d6 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,18 +14,18 @@ jobs: name: Docker actions uses: RMI-PACTA/actions/.github/workflows/docker.yml@main - # test: - # name: Test - # uses: ./.github/workflows/test.yml - # needs: [docker] - # secrets: inherit - # strategy: - # fail-fast: false - # matrix: - # config-name: - # - default_2022Q4 - # - default_2023Q4 - # - full_params_2023Q4 - # with: - # full-image-name: ${{ needs.docker.outputs.full-image-name }} - # config-name: ${{ matrix.config-name }} + test: + name: Test + uses: ./.github/workflows/test.yml + needs: [docker] + secrets: inherit + strategy: + fail-fast: false + matrix: + config-name: + - default_2022Q4 + - default_2023Q4 + - full_params_2023Q4 + with: + full-image-name: ${{ needs.docker.outputs.full-image-name }} + config-name: ${{ matrix.config-name }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..c7e74bb --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,133 @@ +--- +name: Test docker image + +on: + workflow_call: + inputs: + full-image-name: + required: true + type: string + config-name: + required: true + type: string + results-url: + description: azure blob store path for results + required: false + default: "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results" + type: string + +jobs: + test: + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + + - name: Checkout workflow.pacta + uses: actions/checkout@v4 + + - name: Prepare environment + id: prepare + env: + CONFIG_NAME: ${{ inputs.config-name }} + run: | + config_file="tests/config/$CONFIG_NAME.json" + echo "config_file: $config_file" + + HOLDINGS_DATE="$(jq -rc '.holdingsDate' $config_file)" + echo "holdings-date=$HOLDINGS_DATE" + echo "holdings-date=$HOLDINGS_DATE" >> "$GITHUB_OUTPUT" + + PACTA_DATA_URL="$(jq -rc '.pactaDataURL' $config_file)" + echo "pacta-data-url=$PACTA_DATA_URL" + echo "pacta-data-url=$PACTA_DATA_URL" >> "$GITHUB_OUTPUT" + + # includes handling for null/missing keys + PARAMETERS="$(jq -rc '.parameters | select( . != null )' $config_file)" + echo "parameters=$PARAMETERS" + echo "parameters=$PARAMETERS" >> "$GITHUB_OUTPUT" + + # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended + - name: Azure Login + uses: azure/login@v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + + # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version + - name: Download pacta-data + uses: azure/CLI@v2 + env: + PACTA_DATA_URL: ${{ steps.prepare.outputs.pacta-data-url }} + with: + # azcliversion: 2.30.0 + inlineScript: | + echo "pacta_data_dir: $PACTA_DATA_DATE_DIR" + az storage copy \ + --source "$PACTA_DATA_URL/*" \ + --destination "pacta-data" \ + --recursive \ + --exclude-pattern "*.sqlite" + + - name: Run PACTA Analysis + env: + FULL_IMAGE_NAME: ${{ inputs.full-image-name }} + PORTFOLIO_DIR: portfolios + OUTPUT_DIR: output_dir + PACTA_DATA_DIR: pacta-data + WORKSPACE: ${{ github.workspace }} + PARAMETERS: ${{ steps.prepare.outputs.parameters }} + run: | + chmod -R 777 "${WORKSPACE}/${OUTPUT_DIR}" + docker run \ + --network none \ + --env LOG_LEVEL=DEBUG \ + --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ + --mount type=bind,source=${WORKSPACE}/${OUTPUT_DIR},target=/mnt/output_dir \ + --mount type=bind,source=${WORKSPACE}/${PORTFOLIO_DIR},target=/mnt/portfolios \ + ghcr.io/rmi-pacta/workflow.pacta:main \ + $PARAMETERS + + - name: Run Docker Image + env: + FULL_IMAGE_NAME: ${{ inputs.full-image-name }} + PORTFOLIO_DIR: portfolios + OUTPUT_DIR: output_dir + PACTA_DATA_DIR: pacta-data + WORKSPACE: ${{ github.workspace }} + PARAMETERS: ${{ steps.prepare.outputs.parameters }} + run: | + chmod -R 777 "${WORKSPACE}/${OUTPUT_DIR}" + docker run \ + --network none \ + --env LOG_LEVEL=DEBUG \ + --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ + --mount type=bind,source=${WORKSPACE}/${OUTPUT_DIR},target=/mnt/output_dir \ + --mount type=bind,source=${WORKSPACE}/${PORTFOLIO_DIR},target=/mnt/portfolios \ + $FULL_IMAGE_NAME + # $FULL_IMAGE_NAME \ + # $PARAMETERS + + - name: List outputs + run: | + ls -lR output_dir + + # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version + - name: Upload results to blob store + uses: azure/CLI@v2 + env: + CONFIG_NAME: ${{ inputs.config-name }} + GITHUB_REF_NAME: ${{ github.ref_name}} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + GITHUB_RUN_NUMBER: ${{ github.run_number }} + OUTPUTS_DIR: "output_dir" + RESULTS_URL: ${{ inputs.results-url }} + with: + inlineScript: | + unique_directory="$RESULTS_URL/$GITHUB_REF_NAME/$GITHUB_RUN_NUMBER/$GITHUB_RUN_ATTEMPT/$CONFIG_NAME" + az storage copy \ + --source "$outputs_dir" \ + --destination "$unique_directory" \ + --recursive From 96cf1642e25755b10aad72071642170bb46a6c90 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 21:19:06 +0200 Subject: [PATCH 022/132] Move main script --- pacta_03.R => R/run_pacta_reporting_process.R | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pacta_03.R => R/run_pacta_reporting_process.R (100%) diff --git a/pacta_03.R b/R/run_pacta_reporting_process.R similarity index 100% rename from pacta_03.R rename to R/run_pacta_reporting_process.R From f90ad2e1d322f64bf4d72f870e321f5e4bfbb2a7 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 21:24:51 +0200 Subject: [PATCH 023/132] Call an argumentless function as entrypoint --- R/run_pacta_reporting_process.R | 668 ++++++++++++++++---------------- run-pacta.sh | 2 +- 2 files changed, 336 insertions(+), 334 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index ae9399c..b21c6a1 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,361 +1,363 @@ -suppressPackageStartupMessages({ - library(pacta.portfolio.utils) - library(pacta.portfolio.report) - library(pacta.executive.summary) - library(dplyr) - library(readr) - library(jsonlite) - library(fs) -}) - -# defaulting to WARN to maintain current (silent) behavior. -logger::log_threshold(Sys.getenv("LOG_LEVEL", "WARN")) -logger::log_formatter(logger::formatter_glue) - -# ------------------------------------------------------------------------- - -logger::log_info("Starting portfolio report process") - -logger::log_trace("Determining configuration file path") -cfg_path <- commandArgs(trailingOnly = TRUE) -if (length(cfg_path) == 0 || cfg_path == "") { - logger::log_warn("No configuration file specified, using default") - cfg_path <- "/workflow.pacta.report/input_dir/default_config.json" -} -logger::log_debug("Loading configuration from file: \"{cfg_path}\".") -cfg <- fromJSON(cfg_path) - -# quit if there's no relevant PACTA assets ------------------------------------- - -logger::log_debug("Checking for PACTA relevant data in portfolio results.") -total_portfolio_path <- file.path(cfg$output_dir, "total_portfolio.rds") -if (file.exists(total_portfolio_path)) { - total_portfolio <- readRDS(total_portfolio_path) - logger::log_trace("Checking for PACTA relevant data in file: \"{total_portfolio_path}\".") - quit_if_no_pacta_relevant_data(total_portfolio) -} else { - logger::log_warn("file \"{total_portfolio_path}\" does not exist.") - warning("This is weird... the `total_portfolio.rds` file does not exist in the `30_Processed_inputs` directory.") -} +run_pacta_reporting_process <- function() { + suppressPackageStartupMessages({ + library(pacta.portfolio.utils) + library(pacta.portfolio.report) + library(pacta.executive.summary) + library(dplyr) + library(readr) + library(jsonlite) + library(fs) + }) + + # defaulting to WARN to maintain current (silent) behavior. + logger::log_threshold(Sys.getenv("LOG_LEVEL", "WARN")) + logger::log_formatter(logger::formatter_glue) + + # ------------------------------------------------------------------------- + + logger::log_info("Starting portfolio report process") + + logger::log_trace("Determining configuration file path") + cfg_path <- commandArgs(trailingOnly = TRUE) + if (length(cfg_path) == 0 || cfg_path == "") { + logger::log_warn("No configuration file specified, using default") + cfg_path <- "/workflow.pacta.report/input_dir/default_config.json" + } + logger::log_debug("Loading configuration from file: \"{cfg_path}\".") + cfg <- fromJSON(cfg_path) + + # quit if there's no relevant PACTA assets ------------------------------------- + + logger::log_debug("Checking for PACTA relevant data in portfolio results.") + total_portfolio_path <- file.path(cfg$output_dir, "total_portfolio.rds") + if (file.exists(total_portfolio_path)) { + total_portfolio <- readRDS(total_portfolio_path) + logger::log_trace("Checking for PACTA relevant data in file: \"{total_portfolio_path}\".") + quit_if_no_pacta_relevant_data(total_portfolio) + } else { + logger::log_warn("file \"{total_portfolio_path}\" does not exist.") + warning("This is weird... the `total_portfolio.rds` file does not exist in the `30_Processed_inputs` directory.") + } -# fix parameters --------------------------------------------------------------- + # fix parameters --------------------------------------------------------------- -if (cfg$project_code == "GENERAL") { - logger::log_warn("Overriding language selection to \"EN\" for \"GENERAL\".") - cfg$language_select <- "EN" -} else { - logger::log_trace("Using language selection: \"{cfg$language_select}\".") -} + if (cfg$project_code == "GENERAL") { + logger::log_warn("Overriding language selection to \"EN\" for \"GENERAL\".") + cfg$language_select <- "EN" + } else { + logger::log_trace("Using language selection: \"{cfg$language_select}\".") + } -# load PACTA results ----------------------------------------------------------- + # load PACTA results ----------------------------------------------------------- -logger::log_info("Loading PACTA results.") + logger::log_info("Loading PACTA results.") -readRDS_or_return_alt_data <- function(filepath, alt_return = NULL) { - if (file.exists(filepath)) { - return(readRDS(filepath)) + readRDS_or_return_alt_data <- function(filepath, alt_return = NULL) { + if (file.exists(filepath)) { + return(readRDS(filepath)) + } + alt_return } - alt_return -} -add_inv_and_port_names_if_needed <- function(data) { - if (!inherits(data, "data.frame")) { - return(data) - } + add_inv_and_port_names_if_needed <- function(data) { + if (!inherits(data, "data.frame")) { + return(data) + } - if (!"portfolio_name" %in% names(data)) { - data <- mutate(data, portfolio_name = cfg$portfolio_name, .before = everything()) - } + if (!"portfolio_name" %in% names(data)) { + data <- mutate(data, portfolio_name = cfg$portfolio_name, .before = everything()) + } - if (!"investor_name" %in% names(data)) { - data <- mutate(data, investor_name = cfg$investor_name, .before = everything()) + if (!"investor_name" %in% names(data)) { + data <- mutate(data, investor_name = cfg$investor_name, .before = everything()) + } + + data } - data -} + logger::log_debug("Loading audit file.") + audit_file <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "audit_file.rds"), + alt_return = empty_audit_file() + ) + audit_file <- add_inv_and_port_names_if_needed(audit_file) -logger::log_debug("Loading audit file.") -audit_file <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "audit_file.rds"), - alt_return = empty_audit_file() -) -audit_file <- add_inv_and_port_names_if_needed(audit_file) - -logger::log_debug("Loading portfolio overview.") -portfolio_overview <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "overview_portfolio.rds"), - alt_return = empty_portfolio_overview() -) -portfolio_overview <- add_inv_and_port_names_if_needed(portfolio_overview) - -logger::log_debug("Loading emissions.") -emissions <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "emissions.rds"), - alt_return = empty_emissions_results() -) -emissions <- add_inv_and_port_names_if_needed(emissions) - -logger::log_debug("Loading total portfolio results.") -total_portfolio <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "total_portfolio.rds"), - alt_return = empty_portfolio_results() -) -total_portfolio <- add_inv_and_port_names_if_needed(total_portfolio) - -logger::log_debug("Loading portfolio equity results.") -equity_results_portfolio <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Equity_results_portfolio.rds"), - alt_return = empty_portfolio_results() -) -equity_results_portfolio <- add_inv_and_port_names_if_needed(equity_results_portfolio) - -logger::log_debug("Loading portfolio bonds results.") -bonds_results_portfolio <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Bonds_results_portfolio.rds"), - alt_return = empty_portfolio_results() -) -bonds_results_portfolio <- add_inv_and_port_names_if_needed(bonds_results_portfolio) - -logger::log_debug("Loading company equity results.") -equity_results_company <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Equity_results_company.rds"), - alt_return = empty_company_results() -) -equity_results_company <- add_inv_and_port_names_if_needed(equity_results_company) - -logger::log_debug("Loading company bonds results.") -bonds_results_company <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Bonds_results_company.rds"), - alt_return = empty_company_results() -) -bonds_results_company <- add_inv_and_port_names_if_needed(bonds_results_company) - -logger::log_debug("Loading equity map results.") -equity_results_map <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Equity_results_map.rds"), - alt_return = empty_map_results() -) -equity_results_map <- add_inv_and_port_names_if_needed(equity_results_map) - -logger::log_debug("Loading bonds map results.") -bonds_results_map <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Bonds_results_map.rds"), - alt_return = empty_map_results() -) -bonds_results_map <- add_inv_and_port_names_if_needed(bonds_results_map) - -logger::log_debug("Loading portfolio equity peer results.") -peers_equity_results_portfolio <- readRDS_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio.rds")), - alt_return = empty_portfolio_results() -) - -logger::log_debug("Loading portfolio bonds peer results.") -peers_bonds_results_portfolio <- readRDS_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio.rds")), - alt_return = empty_portfolio_results() -) - -logger::log_debug("Loading index equity peer results.") -peers_equity_results_user <- readRDS_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio_ind.rds")), - alt_return = empty_portfolio_results() -) - -logger::log_debug("Loading index bonds peer results.") -peers_bonds_results_user <- readRDS_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio_ind.rds")), - alt_return = empty_portfolio_results() -) - -logger::log_debug("Loading index equity portfolio results.") -indices_equity_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_equity_results_portfolio.rds")) - -logger::log_debug("Loading index bonds portfolio results.") -indices_bonds_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_bonds_results_portfolio.rds")) - -# create interactive report ---------------------------------------------------- - -logger::log_debug("Preparing to create interactive report.") - -survey_dir <- file.path(cfg$user_results_path, cfg$project_code, "survey") -real_estate_dir <- file.path(cfg$user_results_path, cfg$project_code, "real_estate") -output_dir <- file.path(cfg$output_dir) - -logger::log_debug("Loading data frame label translations.") -dataframe_translations <- readr::read_csv( - system.file("extdata/translation/dataframe_labels.csv", package = "pacta.portfolio.report"), - col_types = cols() -) - -logger::log_debug("Loading data frame header translations.") -header_dictionary <- readr::read_csv( - system.file("extdata/translation/dataframe_headers.csv", package = "pacta.portfolio.report"), - col_types = cols() -) - -logger::log_debug("Loading JavaScript label translations.") -js_translations <- jsonlite::fromJSON( - txt = system.file("extdata/translation/js_labels.json", package = "pacta.portfolio.report") -) - -logger::log_debug("Loading sector order.") -sector_order <- readr::read_csv( - system.file("extdata/sector_order/sector_order.csv", package = "pacta.portfolio.report"), - col_types = cols() -) - -# combine config files to send to create_interactive_report() -logger::log_trace("Defining configs and manifest.") -pacta_data_public_manifest <- - list( - creation_time_date = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$creation_time_date, - outputs_manifest = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$outputs_manifest + logger::log_debug("Loading portfolio overview.") + portfolio_overview <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "overview_portfolio.rds"), + alt_return = empty_portfolio_overview() ) + portfolio_overview <- add_inv_and_port_names_if_needed(portfolio_overview) -configs <- - list( - portfolio_config = cfg, - pacta_data_public_manifest = pacta_data_public_manifest + logger::log_debug("Loading emissions.") + emissions <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "emissions.rds"), + alt_return = empty_emissions_results() ) + emissions <- add_inv_and_port_names_if_needed(emissions) -# workaround a bug in {config} v0.3.2 that only adds "config" class to objects it creates -class(configs$portfolio_config) <- c(class(configs$portfolio_config), "list") - -logger::log_trace("Defining interactive report template paths.") -template_path <- system.file("templates", package = "pacta.portfolio.report") #TODO: generalize this to accept non-builtin templates -template_dir_name <- paste(tolower(cfg$project_report_name), tolower(cfg$language_select), "template", sep = "_") -template_dir <- file.path(template_path, template_dir_name) - -logger::log_info("Creating interactive report.") -create_interactive_report( - template_dir = template_dir, - output_dir = file.path(cfg$output_dir, "report"), - survey_dir = cfg$survey_dir, - real_estate_dir = cfg$real_estate_dir, - language_select = cfg$language_select, - investor_name = cfg$investor_name, - portfolio_name = cfg$portfolio_name, - peer_group = cfg$peer_group, - start_year = cfg$start_year, - select_scenario = cfg$select_scenario, - select_scenario_other = cfg$scenario_other, - portfolio_allocation_method = cfg$portfolio_allocation_method, - scenario_geography = cfg$scenario_geography, - pacta_sectors = cfg$sector_list, - green_techs = cfg$green_techs, - tech_roadmap_sectors = cfg$tech_roadmap_sectors, - pacta_sectors_not_analysed = cfg$pacta_sectors_not_analysed, - audit_file = audit_file, - emissions = emissions, - portfolio_overview = portfolio_overview, - equity_results_portfolio = equity_results_portfolio, - bonds_results_portfolio = bonds_results_portfolio, - equity_results_company = equity_results_company, - bonds_results_company = bonds_results_company, - equity_results_map = equity_results_map, - bonds_results_map = bonds_results_map, - indices_equity_results_portfolio = indices_equity_results_portfolio, - indices_bonds_results_portfolio = indices_bonds_results_portfolio, - peers_equity_results_portfolio = peers_equity_results_portfolio, - peers_bonds_results_portfolio = peers_bonds_results_portfolio, - peers_equity_results_user = peers_equity_results_user, - peers_bonds_results_user = peers_bonds_results_user, - dataframe_translations = dataframe_translations, - js_translations = js_translations, - display_currency = cfg$display_currency, - currency_exchange_value = cfg$currency_exchange_value, - header_dictionary = header_dictionary, - sector_order = sector_order, - configs = configs -) - - -# create executive summary ----------------------------------------------------- -logger::log_debug("Preparing to create executive summary.") - -survey_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "survey")) -real_estate_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "real_estate")) -score_card_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "score_card")) -output_dir <- file.path(cfg$output_dir) -es_dir <- file.path(cfg$output_dir, "executive_summary") -if (!dir.exists(es_dir)) { - dir.create(es_dir, showWarnings = FALSE, recursive = TRUE) -} + logger::log_debug("Loading total portfolio results.") + total_portfolio <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "total_portfolio.rds"), + alt_return = empty_portfolio_results() + ) + total_portfolio <- add_inv_and_port_names_if_needed(total_portfolio) -logger::log_trace("Defining executive summary template paths.") -exec_summary_template_name <- paste0(cfg$project_code, "_", tolower(cfg$language_select), "_exec_summary") -exec_summary_builtin_template_path <- system.file("extdata", exec_summary_template_name, package = "pacta.executive.summary") -invisible(file.copy(exec_summary_builtin_template_path, cfg$output_dir, recursive = TRUE, copy.mode = FALSE)) -exec_summary_template_path <- file.path(cfg$output_dir, exec_summary_template_name) + logger::log_debug("Loading portfolio equity results.") + equity_results_portfolio <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Equity_results_portfolio.rds"), + alt_return = empty_portfolio_results() + ) + equity_results_portfolio <- add_inv_and_port_names_if_needed(equity_results_portfolio) -if (dir.exists(exec_summary_template_path) && (cfg$peer_group %in% c("assetmanager", "bank", "insurance", "pensionfund"))) { - logger::log_debug("Preparing data for executive summary.") - data_aggregated_filtered <- - prep_data_executive_summary( - investor_name = cfg$investor_name, - portfolio_name = cfg$portfolio_name, - peer_group = cfg$peer_group, - start_year = cfg$start_year, - scenario_source = "GECO2021", - scenario_selected = "1.5C-Unif", - scenario_geography = "Global", - equity_market = "GlobalMarket", - portfolio_allocation_method_equity = "portfolio_weight", - portfolio_allocation_method_bonds = "portfolio_weight", - green_techs = c( - "RenewablesCap", - "HydroCap", - "NuclearCap", - "Hybrid", - "Electric", - "FuelCell", - "Hybrid_HDV", - "Electric_HDV", - "FuelCell_HDV", - "Electric Arc Furnace" - ), - equity_results_portfolio = equity_results_portfolio, - bonds_results_portfolio = bonds_results_portfolio, - peers_equity_results_aggregated = peers_equity_results_portfolio, - peers_bonds_results_aggregated = peers_bonds_results_portfolio, - peers_equity_results_individual = peers_equity_results_user, - peers_bonds_results_individual = peers_bonds_results_user, - indices_equity_results_portfolio = indices_equity_results_portfolio, - indices_bonds_results_portfolio = indices_bonds_results_portfolio, - audit_file = audit_file, - emissions_portfolio = emissions, - score_card_dir = score_card_dir + logger::log_debug("Loading portfolio bonds results.") + bonds_results_portfolio <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Bonds_results_portfolio.rds"), + alt_return = empty_portfolio_results() + ) + bonds_results_portfolio <- add_inv_and_port_names_if_needed(bonds_results_portfolio) + + logger::log_debug("Loading company equity results.") + equity_results_company <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Equity_results_company.rds"), + alt_return = empty_company_results() + ) + equity_results_company <- add_inv_and_port_names_if_needed(equity_results_company) + + logger::log_debug("Loading company bonds results.") + bonds_results_company <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Bonds_results_company.rds"), + alt_return = empty_company_results() + ) + bonds_results_company <- add_inv_and_port_names_if_needed(bonds_results_company) + + logger::log_debug("Loading equity map results.") + equity_results_map <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Equity_results_map.rds"), + alt_return = empty_map_results() + ) + equity_results_map <- add_inv_and_port_names_if_needed(equity_results_map) + + logger::log_debug("Loading bonds map results.") + bonds_results_map <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Bonds_results_map.rds"), + alt_return = empty_map_results() + ) + bonds_results_map <- add_inv_and_port_names_if_needed(bonds_results_map) + + logger::log_debug("Loading portfolio equity peer results.") + peers_equity_results_portfolio <- readRDS_or_return_alt_data( + filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio.rds")), + alt_return = empty_portfolio_results() + ) + + logger::log_debug("Loading portfolio bonds peer results.") + peers_bonds_results_portfolio <- readRDS_or_return_alt_data( + filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio.rds")), + alt_return = empty_portfolio_results() + ) + + logger::log_debug("Loading index equity peer results.") + peers_equity_results_user <- readRDS_or_return_alt_data( + filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio_ind.rds")), + alt_return = empty_portfolio_results() + ) + + logger::log_debug("Loading index bonds peer results.") + peers_bonds_results_user <- readRDS_or_return_alt_data( + filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio_ind.rds")), + alt_return = empty_portfolio_results() + ) + + logger::log_debug("Loading index equity portfolio results.") + indices_equity_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_equity_results_portfolio.rds")) + + logger::log_debug("Loading index bonds portfolio results.") + indices_bonds_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_bonds_results_portfolio.rds")) + + # create interactive report ---------------------------------------------------- + + logger::log_debug("Preparing to create interactive report.") + + survey_dir <- file.path(cfg$user_results_path, cfg$project_code, "survey") + real_estate_dir <- file.path(cfg$user_results_path, cfg$project_code, "real_estate") + output_dir <- file.path(cfg$output_dir) + + logger::log_debug("Loading data frame label translations.") + dataframe_translations <- readr::read_csv( + system.file("extdata/translation/dataframe_labels.csv", package = "pacta.portfolio.report"), + col_types = cols() + ) + + logger::log_debug("Loading data frame header translations.") + header_dictionary <- readr::read_csv( + system.file("extdata/translation/dataframe_headers.csv", package = "pacta.portfolio.report"), + col_types = cols() + ) + + logger::log_debug("Loading JavaScript label translations.") + js_translations <- jsonlite::fromJSON( + txt = system.file("extdata/translation/js_labels.json", package = "pacta.portfolio.report") + ) + + logger::log_debug("Loading sector order.") + sector_order <- readr::read_csv( + system.file("extdata/sector_order/sector_order.csv", package = "pacta.portfolio.report"), + col_types = cols() + ) + + # combine config files to send to create_interactive_report() + logger::log_trace("Defining configs and manifest.") + pacta_data_public_manifest <- + list( + creation_time_date = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$creation_time_date, + outputs_manifest = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$outputs_manifest + ) + + configs <- + list( + portfolio_config = cfg, + pacta_data_public_manifest = pacta_data_public_manifest ) - logger::log_trace("Checking for real estate data.") - real_estate_flag <- (length(list.files(real_estate_dir)) > 0) - - logger::log_info("Creating executive summary.") - render_executive_summary( - data = data_aggregated_filtered, - language = cfg$language_select, - output_dir = es_dir, - exec_summary_dir = exec_summary_template_path, - survey_dir = survey_dir, - real_estate_dir = real_estate_dir, - real_estate_flag = real_estate_flag, - score_card_dir = score_card_dir, - file_name = "template.Rmd", + # workaround a bug in {config} v0.3.2 that only adds "config" class to objects it creates + class(configs$portfolio_config) <- c(class(configs$portfolio_config), "list") + + logger::log_trace("Defining interactive report template paths.") + template_path <- system.file("templates", package = "pacta.portfolio.report") #TODO: generalize this to accept non-builtin templates + template_dir_name <- paste(tolower(cfg$project_report_name), tolower(cfg$language_select), "template", sep = "_") + template_dir <- file.path(template_path, template_dir_name) + + logger::log_info("Creating interactive report.") + create_interactive_report( + template_dir = template_dir, + output_dir = file.path(cfg$output_dir, "report"), + survey_dir = cfg$survey_dir, + real_estate_dir = cfg$real_estate_dir, + language_select = cfg$language_select, investor_name = cfg$investor_name, portfolio_name = cfg$portfolio_name, peer_group = cfg$peer_group, - total_portfolio = total_portfolio, - scenario_selected = "1.5C-Unif", + start_year = cfg$start_year, + select_scenario = cfg$select_scenario, + select_scenario_other = cfg$scenario_other, + portfolio_allocation_method = cfg$portfolio_allocation_method, + scenario_geography = cfg$scenario_geography, + pacta_sectors = cfg$sector_list, + green_techs = cfg$green_techs, + tech_roadmap_sectors = cfg$tech_roadmap_sectors, + pacta_sectors_not_analysed = cfg$pacta_sectors_not_analysed, + audit_file = audit_file, + emissions = emissions, + portfolio_overview = portfolio_overview, + equity_results_portfolio = equity_results_portfolio, + bonds_results_portfolio = bonds_results_portfolio, + equity_results_company = equity_results_company, + bonds_results_company = bonds_results_company, + equity_results_map = equity_results_map, + bonds_results_map = bonds_results_map, + indices_equity_results_portfolio = indices_equity_results_portfolio, + indices_bonds_results_portfolio = indices_bonds_results_portfolio, + peers_equity_results_portfolio = peers_equity_results_portfolio, + peers_bonds_results_portfolio = peers_bonds_results_portfolio, + peers_equity_results_user = peers_equity_results_user, + peers_bonds_results_user = peers_bonds_results_user, + dataframe_translations = dataframe_translations, + js_translations = js_translations, + display_currency = cfg$display_currency, currency_exchange_value = cfg$currency_exchange_value, - log_dir = cfg$output_dir + header_dictionary = header_dictionary, + sector_order = sector_order, + configs = configs ) -} else { - # this is required for the online tool to know that the process has been completed. - logger::log_debug("No executive summary created.") - invisible(file.copy(blank_pdf(), es_dir)) -} -logger::log_info("Portfolio report finished.") + + # create executive summary ----------------------------------------------------- + logger::log_debug("Preparing to create executive summary.") + + survey_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "survey")) + real_estate_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "real_estate")) + score_card_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "score_card")) + output_dir <- file.path(cfg$output_dir) + es_dir <- file.path(cfg$output_dir, "executive_summary") + if (!dir.exists(es_dir)) { + dir.create(es_dir, showWarnings = FALSE, recursive = TRUE) + } + + logger::log_trace("Defining executive summary template paths.") + exec_summary_template_name <- paste0(cfg$project_code, "_", tolower(cfg$language_select), "_exec_summary") + exec_summary_builtin_template_path <- system.file("extdata", exec_summary_template_name, package = "pacta.executive.summary") + invisible(file.copy(exec_summary_builtin_template_path, cfg$output_dir, recursive = TRUE, copy.mode = FALSE)) + exec_summary_template_path <- file.path(cfg$output_dir, exec_summary_template_name) + + if (dir.exists(exec_summary_template_path) && (cfg$peer_group %in% c("assetmanager", "bank", "insurance", "pensionfund"))) { + logger::log_debug("Preparing data for executive summary.") + data_aggregated_filtered <- + prep_data_executive_summary( + investor_name = cfg$investor_name, + portfolio_name = cfg$portfolio_name, + peer_group = cfg$peer_group, + start_year = cfg$start_year, + scenario_source = "GECO2021", + scenario_selected = "1.5C-Unif", + scenario_geography = "Global", + equity_market = "GlobalMarket", + portfolio_allocation_method_equity = "portfolio_weight", + portfolio_allocation_method_bonds = "portfolio_weight", + green_techs = c( + "RenewablesCap", + "HydroCap", + "NuclearCap", + "Hybrid", + "Electric", + "FuelCell", + "Hybrid_HDV", + "Electric_HDV", + "FuelCell_HDV", + "Electric Arc Furnace" + ), + equity_results_portfolio = equity_results_portfolio, + bonds_results_portfolio = bonds_results_portfolio, + peers_equity_results_aggregated = peers_equity_results_portfolio, + peers_bonds_results_aggregated = peers_bonds_results_portfolio, + peers_equity_results_individual = peers_equity_results_user, + peers_bonds_results_individual = peers_bonds_results_user, + indices_equity_results_portfolio = indices_equity_results_portfolio, + indices_bonds_results_portfolio = indices_bonds_results_portfolio, + audit_file = audit_file, + emissions_portfolio = emissions, + score_card_dir = score_card_dir + ) + + logger::log_trace("Checking for real estate data.") + real_estate_flag <- (length(list.files(real_estate_dir)) > 0) + + logger::log_info("Creating executive summary.") + render_executive_summary( + data = data_aggregated_filtered, + language = cfg$language_select, + output_dir = es_dir, + exec_summary_dir = exec_summary_template_path, + survey_dir = survey_dir, + real_estate_dir = real_estate_dir, + real_estate_flag = real_estate_flag, + score_card_dir = score_card_dir, + file_name = "template.Rmd", + investor_name = cfg$investor_name, + portfolio_name = cfg$portfolio_name, + peer_group = cfg$peer_group, + total_portfolio = total_portfolio, + scenario_selected = "1.5C-Unif", + currency_exchange_value = cfg$currency_exchange_value, + log_dir = cfg$output_dir + ) + } else { + # this is required for the online tool to know that the process has been completed. + logger::log_debug("No executive summary created.") + invisible(file.copy(blank_pdf(), es_dir)) + } + + logger::log_info("Portfolio report finished.") +} diff --git a/run-pacta.sh b/run-pacta.sh index 9b77174..dbcc287 100755 --- a/run-pacta.sh +++ b/run-pacta.sh @@ -3,4 +3,4 @@ # Set permissions so that new files can be deleted/overwritten outside docker umask 000 -Rscript --vanilla /workflow.pacta.report/pacta_03.R "${1}" \ +Rscript --vanilla -e "workflow.pacta.report:::run_pacta_reporting_process()" "${1}" \ From 345a496b9dd88962071427a541efa5d2fae810d0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 21:19:06 +0200 Subject: [PATCH 024/132] Move main script --- pacta_03.R => R/run_pacta_reporting_process.R | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pacta_03.R => R/run_pacta_reporting_process.R (100%) diff --git a/pacta_03.R b/R/run_pacta_reporting_process.R similarity index 100% rename from pacta_03.R rename to R/run_pacta_reporting_process.R From bf286deed99b780f88d53d78e22529ccfa96a7e6 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 21:24:51 +0200 Subject: [PATCH 025/132] Call an argumentless function as entrypoint --- R/run_pacta_reporting_process.R | 668 ++++++++++++++++---------------- run-pacta.sh | 2 +- 2 files changed, 336 insertions(+), 334 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index ae9399c..b21c6a1 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,361 +1,363 @@ -suppressPackageStartupMessages({ - library(pacta.portfolio.utils) - library(pacta.portfolio.report) - library(pacta.executive.summary) - library(dplyr) - library(readr) - library(jsonlite) - library(fs) -}) - -# defaulting to WARN to maintain current (silent) behavior. -logger::log_threshold(Sys.getenv("LOG_LEVEL", "WARN")) -logger::log_formatter(logger::formatter_glue) - -# ------------------------------------------------------------------------- - -logger::log_info("Starting portfolio report process") - -logger::log_trace("Determining configuration file path") -cfg_path <- commandArgs(trailingOnly = TRUE) -if (length(cfg_path) == 0 || cfg_path == "") { - logger::log_warn("No configuration file specified, using default") - cfg_path <- "/workflow.pacta.report/input_dir/default_config.json" -} -logger::log_debug("Loading configuration from file: \"{cfg_path}\".") -cfg <- fromJSON(cfg_path) - -# quit if there's no relevant PACTA assets ------------------------------------- - -logger::log_debug("Checking for PACTA relevant data in portfolio results.") -total_portfolio_path <- file.path(cfg$output_dir, "total_portfolio.rds") -if (file.exists(total_portfolio_path)) { - total_portfolio <- readRDS(total_portfolio_path) - logger::log_trace("Checking for PACTA relevant data in file: \"{total_portfolio_path}\".") - quit_if_no_pacta_relevant_data(total_portfolio) -} else { - logger::log_warn("file \"{total_portfolio_path}\" does not exist.") - warning("This is weird... the `total_portfolio.rds` file does not exist in the `30_Processed_inputs` directory.") -} +run_pacta_reporting_process <- function() { + suppressPackageStartupMessages({ + library(pacta.portfolio.utils) + library(pacta.portfolio.report) + library(pacta.executive.summary) + library(dplyr) + library(readr) + library(jsonlite) + library(fs) + }) + + # defaulting to WARN to maintain current (silent) behavior. + logger::log_threshold(Sys.getenv("LOG_LEVEL", "WARN")) + logger::log_formatter(logger::formatter_glue) + + # ------------------------------------------------------------------------- + + logger::log_info("Starting portfolio report process") + + logger::log_trace("Determining configuration file path") + cfg_path <- commandArgs(trailingOnly = TRUE) + if (length(cfg_path) == 0 || cfg_path == "") { + logger::log_warn("No configuration file specified, using default") + cfg_path <- "/workflow.pacta.report/input_dir/default_config.json" + } + logger::log_debug("Loading configuration from file: \"{cfg_path}\".") + cfg <- fromJSON(cfg_path) + + # quit if there's no relevant PACTA assets ------------------------------------- + + logger::log_debug("Checking for PACTA relevant data in portfolio results.") + total_portfolio_path <- file.path(cfg$output_dir, "total_portfolio.rds") + if (file.exists(total_portfolio_path)) { + total_portfolio <- readRDS(total_portfolio_path) + logger::log_trace("Checking for PACTA relevant data in file: \"{total_portfolio_path}\".") + quit_if_no_pacta_relevant_data(total_portfolio) + } else { + logger::log_warn("file \"{total_portfolio_path}\" does not exist.") + warning("This is weird... the `total_portfolio.rds` file does not exist in the `30_Processed_inputs` directory.") + } -# fix parameters --------------------------------------------------------------- + # fix parameters --------------------------------------------------------------- -if (cfg$project_code == "GENERAL") { - logger::log_warn("Overriding language selection to \"EN\" for \"GENERAL\".") - cfg$language_select <- "EN" -} else { - logger::log_trace("Using language selection: \"{cfg$language_select}\".") -} + if (cfg$project_code == "GENERAL") { + logger::log_warn("Overriding language selection to \"EN\" for \"GENERAL\".") + cfg$language_select <- "EN" + } else { + logger::log_trace("Using language selection: \"{cfg$language_select}\".") + } -# load PACTA results ----------------------------------------------------------- + # load PACTA results ----------------------------------------------------------- -logger::log_info("Loading PACTA results.") + logger::log_info("Loading PACTA results.") -readRDS_or_return_alt_data <- function(filepath, alt_return = NULL) { - if (file.exists(filepath)) { - return(readRDS(filepath)) + readRDS_or_return_alt_data <- function(filepath, alt_return = NULL) { + if (file.exists(filepath)) { + return(readRDS(filepath)) + } + alt_return } - alt_return -} -add_inv_and_port_names_if_needed <- function(data) { - if (!inherits(data, "data.frame")) { - return(data) - } + add_inv_and_port_names_if_needed <- function(data) { + if (!inherits(data, "data.frame")) { + return(data) + } - if (!"portfolio_name" %in% names(data)) { - data <- mutate(data, portfolio_name = cfg$portfolio_name, .before = everything()) - } + if (!"portfolio_name" %in% names(data)) { + data <- mutate(data, portfolio_name = cfg$portfolio_name, .before = everything()) + } - if (!"investor_name" %in% names(data)) { - data <- mutate(data, investor_name = cfg$investor_name, .before = everything()) + if (!"investor_name" %in% names(data)) { + data <- mutate(data, investor_name = cfg$investor_name, .before = everything()) + } + + data } - data -} + logger::log_debug("Loading audit file.") + audit_file <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "audit_file.rds"), + alt_return = empty_audit_file() + ) + audit_file <- add_inv_and_port_names_if_needed(audit_file) -logger::log_debug("Loading audit file.") -audit_file <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "audit_file.rds"), - alt_return = empty_audit_file() -) -audit_file <- add_inv_and_port_names_if_needed(audit_file) - -logger::log_debug("Loading portfolio overview.") -portfolio_overview <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "overview_portfolio.rds"), - alt_return = empty_portfolio_overview() -) -portfolio_overview <- add_inv_and_port_names_if_needed(portfolio_overview) - -logger::log_debug("Loading emissions.") -emissions <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "emissions.rds"), - alt_return = empty_emissions_results() -) -emissions <- add_inv_and_port_names_if_needed(emissions) - -logger::log_debug("Loading total portfolio results.") -total_portfolio <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "total_portfolio.rds"), - alt_return = empty_portfolio_results() -) -total_portfolio <- add_inv_and_port_names_if_needed(total_portfolio) - -logger::log_debug("Loading portfolio equity results.") -equity_results_portfolio <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Equity_results_portfolio.rds"), - alt_return = empty_portfolio_results() -) -equity_results_portfolio <- add_inv_and_port_names_if_needed(equity_results_portfolio) - -logger::log_debug("Loading portfolio bonds results.") -bonds_results_portfolio <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Bonds_results_portfolio.rds"), - alt_return = empty_portfolio_results() -) -bonds_results_portfolio <- add_inv_and_port_names_if_needed(bonds_results_portfolio) - -logger::log_debug("Loading company equity results.") -equity_results_company <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Equity_results_company.rds"), - alt_return = empty_company_results() -) -equity_results_company <- add_inv_and_port_names_if_needed(equity_results_company) - -logger::log_debug("Loading company bonds results.") -bonds_results_company <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Bonds_results_company.rds"), - alt_return = empty_company_results() -) -bonds_results_company <- add_inv_and_port_names_if_needed(bonds_results_company) - -logger::log_debug("Loading equity map results.") -equity_results_map <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Equity_results_map.rds"), - alt_return = empty_map_results() -) -equity_results_map <- add_inv_and_port_names_if_needed(equity_results_map) - -logger::log_debug("Loading bonds map results.") -bonds_results_map <- readRDS_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Bonds_results_map.rds"), - alt_return = empty_map_results() -) -bonds_results_map <- add_inv_and_port_names_if_needed(bonds_results_map) - -logger::log_debug("Loading portfolio equity peer results.") -peers_equity_results_portfolio <- readRDS_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio.rds")), - alt_return = empty_portfolio_results() -) - -logger::log_debug("Loading portfolio bonds peer results.") -peers_bonds_results_portfolio <- readRDS_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio.rds")), - alt_return = empty_portfolio_results() -) - -logger::log_debug("Loading index equity peer results.") -peers_equity_results_user <- readRDS_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio_ind.rds")), - alt_return = empty_portfolio_results() -) - -logger::log_debug("Loading index bonds peer results.") -peers_bonds_results_user <- readRDS_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio_ind.rds")), - alt_return = empty_portfolio_results() -) - -logger::log_debug("Loading index equity portfolio results.") -indices_equity_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_equity_results_portfolio.rds")) - -logger::log_debug("Loading index bonds portfolio results.") -indices_bonds_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_bonds_results_portfolio.rds")) - -# create interactive report ---------------------------------------------------- - -logger::log_debug("Preparing to create interactive report.") - -survey_dir <- file.path(cfg$user_results_path, cfg$project_code, "survey") -real_estate_dir <- file.path(cfg$user_results_path, cfg$project_code, "real_estate") -output_dir <- file.path(cfg$output_dir) - -logger::log_debug("Loading data frame label translations.") -dataframe_translations <- readr::read_csv( - system.file("extdata/translation/dataframe_labels.csv", package = "pacta.portfolio.report"), - col_types = cols() -) - -logger::log_debug("Loading data frame header translations.") -header_dictionary <- readr::read_csv( - system.file("extdata/translation/dataframe_headers.csv", package = "pacta.portfolio.report"), - col_types = cols() -) - -logger::log_debug("Loading JavaScript label translations.") -js_translations <- jsonlite::fromJSON( - txt = system.file("extdata/translation/js_labels.json", package = "pacta.portfolio.report") -) - -logger::log_debug("Loading sector order.") -sector_order <- readr::read_csv( - system.file("extdata/sector_order/sector_order.csv", package = "pacta.portfolio.report"), - col_types = cols() -) - -# combine config files to send to create_interactive_report() -logger::log_trace("Defining configs and manifest.") -pacta_data_public_manifest <- - list( - creation_time_date = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$creation_time_date, - outputs_manifest = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$outputs_manifest + logger::log_debug("Loading portfolio overview.") + portfolio_overview <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "overview_portfolio.rds"), + alt_return = empty_portfolio_overview() ) + portfolio_overview <- add_inv_and_port_names_if_needed(portfolio_overview) -configs <- - list( - portfolio_config = cfg, - pacta_data_public_manifest = pacta_data_public_manifest + logger::log_debug("Loading emissions.") + emissions <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "emissions.rds"), + alt_return = empty_emissions_results() ) + emissions <- add_inv_and_port_names_if_needed(emissions) -# workaround a bug in {config} v0.3.2 that only adds "config" class to objects it creates -class(configs$portfolio_config) <- c(class(configs$portfolio_config), "list") - -logger::log_trace("Defining interactive report template paths.") -template_path <- system.file("templates", package = "pacta.portfolio.report") #TODO: generalize this to accept non-builtin templates -template_dir_name <- paste(tolower(cfg$project_report_name), tolower(cfg$language_select), "template", sep = "_") -template_dir <- file.path(template_path, template_dir_name) - -logger::log_info("Creating interactive report.") -create_interactive_report( - template_dir = template_dir, - output_dir = file.path(cfg$output_dir, "report"), - survey_dir = cfg$survey_dir, - real_estate_dir = cfg$real_estate_dir, - language_select = cfg$language_select, - investor_name = cfg$investor_name, - portfolio_name = cfg$portfolio_name, - peer_group = cfg$peer_group, - start_year = cfg$start_year, - select_scenario = cfg$select_scenario, - select_scenario_other = cfg$scenario_other, - portfolio_allocation_method = cfg$portfolio_allocation_method, - scenario_geography = cfg$scenario_geography, - pacta_sectors = cfg$sector_list, - green_techs = cfg$green_techs, - tech_roadmap_sectors = cfg$tech_roadmap_sectors, - pacta_sectors_not_analysed = cfg$pacta_sectors_not_analysed, - audit_file = audit_file, - emissions = emissions, - portfolio_overview = portfolio_overview, - equity_results_portfolio = equity_results_portfolio, - bonds_results_portfolio = bonds_results_portfolio, - equity_results_company = equity_results_company, - bonds_results_company = bonds_results_company, - equity_results_map = equity_results_map, - bonds_results_map = bonds_results_map, - indices_equity_results_portfolio = indices_equity_results_portfolio, - indices_bonds_results_portfolio = indices_bonds_results_portfolio, - peers_equity_results_portfolio = peers_equity_results_portfolio, - peers_bonds_results_portfolio = peers_bonds_results_portfolio, - peers_equity_results_user = peers_equity_results_user, - peers_bonds_results_user = peers_bonds_results_user, - dataframe_translations = dataframe_translations, - js_translations = js_translations, - display_currency = cfg$display_currency, - currency_exchange_value = cfg$currency_exchange_value, - header_dictionary = header_dictionary, - sector_order = sector_order, - configs = configs -) - - -# create executive summary ----------------------------------------------------- -logger::log_debug("Preparing to create executive summary.") - -survey_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "survey")) -real_estate_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "real_estate")) -score_card_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "score_card")) -output_dir <- file.path(cfg$output_dir) -es_dir <- file.path(cfg$output_dir, "executive_summary") -if (!dir.exists(es_dir)) { - dir.create(es_dir, showWarnings = FALSE, recursive = TRUE) -} + logger::log_debug("Loading total portfolio results.") + total_portfolio <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "total_portfolio.rds"), + alt_return = empty_portfolio_results() + ) + total_portfolio <- add_inv_and_port_names_if_needed(total_portfolio) -logger::log_trace("Defining executive summary template paths.") -exec_summary_template_name <- paste0(cfg$project_code, "_", tolower(cfg$language_select), "_exec_summary") -exec_summary_builtin_template_path <- system.file("extdata", exec_summary_template_name, package = "pacta.executive.summary") -invisible(file.copy(exec_summary_builtin_template_path, cfg$output_dir, recursive = TRUE, copy.mode = FALSE)) -exec_summary_template_path <- file.path(cfg$output_dir, exec_summary_template_name) + logger::log_debug("Loading portfolio equity results.") + equity_results_portfolio <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Equity_results_portfolio.rds"), + alt_return = empty_portfolio_results() + ) + equity_results_portfolio <- add_inv_and_port_names_if_needed(equity_results_portfolio) -if (dir.exists(exec_summary_template_path) && (cfg$peer_group %in% c("assetmanager", "bank", "insurance", "pensionfund"))) { - logger::log_debug("Preparing data for executive summary.") - data_aggregated_filtered <- - prep_data_executive_summary( - investor_name = cfg$investor_name, - portfolio_name = cfg$portfolio_name, - peer_group = cfg$peer_group, - start_year = cfg$start_year, - scenario_source = "GECO2021", - scenario_selected = "1.5C-Unif", - scenario_geography = "Global", - equity_market = "GlobalMarket", - portfolio_allocation_method_equity = "portfolio_weight", - portfolio_allocation_method_bonds = "portfolio_weight", - green_techs = c( - "RenewablesCap", - "HydroCap", - "NuclearCap", - "Hybrid", - "Electric", - "FuelCell", - "Hybrid_HDV", - "Electric_HDV", - "FuelCell_HDV", - "Electric Arc Furnace" - ), - equity_results_portfolio = equity_results_portfolio, - bonds_results_portfolio = bonds_results_portfolio, - peers_equity_results_aggregated = peers_equity_results_portfolio, - peers_bonds_results_aggregated = peers_bonds_results_portfolio, - peers_equity_results_individual = peers_equity_results_user, - peers_bonds_results_individual = peers_bonds_results_user, - indices_equity_results_portfolio = indices_equity_results_portfolio, - indices_bonds_results_portfolio = indices_bonds_results_portfolio, - audit_file = audit_file, - emissions_portfolio = emissions, - score_card_dir = score_card_dir + logger::log_debug("Loading portfolio bonds results.") + bonds_results_portfolio <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Bonds_results_portfolio.rds"), + alt_return = empty_portfolio_results() + ) + bonds_results_portfolio <- add_inv_and_port_names_if_needed(bonds_results_portfolio) + + logger::log_debug("Loading company equity results.") + equity_results_company <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Equity_results_company.rds"), + alt_return = empty_company_results() + ) + equity_results_company <- add_inv_and_port_names_if_needed(equity_results_company) + + logger::log_debug("Loading company bonds results.") + bonds_results_company <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Bonds_results_company.rds"), + alt_return = empty_company_results() + ) + bonds_results_company <- add_inv_and_port_names_if_needed(bonds_results_company) + + logger::log_debug("Loading equity map results.") + equity_results_map <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Equity_results_map.rds"), + alt_return = empty_map_results() + ) + equity_results_map <- add_inv_and_port_names_if_needed(equity_results_map) + + logger::log_debug("Loading bonds map results.") + bonds_results_map <- readRDS_or_return_alt_data( + filepath = file.path(cfg$output_dir, "Bonds_results_map.rds"), + alt_return = empty_map_results() + ) + bonds_results_map <- add_inv_and_port_names_if_needed(bonds_results_map) + + logger::log_debug("Loading portfolio equity peer results.") + peers_equity_results_portfolio <- readRDS_or_return_alt_data( + filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio.rds")), + alt_return = empty_portfolio_results() + ) + + logger::log_debug("Loading portfolio bonds peer results.") + peers_bonds_results_portfolio <- readRDS_or_return_alt_data( + filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio.rds")), + alt_return = empty_portfolio_results() + ) + + logger::log_debug("Loading index equity peer results.") + peers_equity_results_user <- readRDS_or_return_alt_data( + filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio_ind.rds")), + alt_return = empty_portfolio_results() + ) + + logger::log_debug("Loading index bonds peer results.") + peers_bonds_results_user <- readRDS_or_return_alt_data( + filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio_ind.rds")), + alt_return = empty_portfolio_results() + ) + + logger::log_debug("Loading index equity portfolio results.") + indices_equity_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_equity_results_portfolio.rds")) + + logger::log_debug("Loading index bonds portfolio results.") + indices_bonds_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_bonds_results_portfolio.rds")) + + # create interactive report ---------------------------------------------------- + + logger::log_debug("Preparing to create interactive report.") + + survey_dir <- file.path(cfg$user_results_path, cfg$project_code, "survey") + real_estate_dir <- file.path(cfg$user_results_path, cfg$project_code, "real_estate") + output_dir <- file.path(cfg$output_dir) + + logger::log_debug("Loading data frame label translations.") + dataframe_translations <- readr::read_csv( + system.file("extdata/translation/dataframe_labels.csv", package = "pacta.portfolio.report"), + col_types = cols() + ) + + logger::log_debug("Loading data frame header translations.") + header_dictionary <- readr::read_csv( + system.file("extdata/translation/dataframe_headers.csv", package = "pacta.portfolio.report"), + col_types = cols() + ) + + logger::log_debug("Loading JavaScript label translations.") + js_translations <- jsonlite::fromJSON( + txt = system.file("extdata/translation/js_labels.json", package = "pacta.portfolio.report") + ) + + logger::log_debug("Loading sector order.") + sector_order <- readr::read_csv( + system.file("extdata/sector_order/sector_order.csv", package = "pacta.portfolio.report"), + col_types = cols() + ) + + # combine config files to send to create_interactive_report() + logger::log_trace("Defining configs and manifest.") + pacta_data_public_manifest <- + list( + creation_time_date = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$creation_time_date, + outputs_manifest = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$outputs_manifest + ) + + configs <- + list( + portfolio_config = cfg, + pacta_data_public_manifest = pacta_data_public_manifest ) - logger::log_trace("Checking for real estate data.") - real_estate_flag <- (length(list.files(real_estate_dir)) > 0) - - logger::log_info("Creating executive summary.") - render_executive_summary( - data = data_aggregated_filtered, - language = cfg$language_select, - output_dir = es_dir, - exec_summary_dir = exec_summary_template_path, - survey_dir = survey_dir, - real_estate_dir = real_estate_dir, - real_estate_flag = real_estate_flag, - score_card_dir = score_card_dir, - file_name = "template.Rmd", + # workaround a bug in {config} v0.3.2 that only adds "config" class to objects it creates + class(configs$portfolio_config) <- c(class(configs$portfolio_config), "list") + + logger::log_trace("Defining interactive report template paths.") + template_path <- system.file("templates", package = "pacta.portfolio.report") #TODO: generalize this to accept non-builtin templates + template_dir_name <- paste(tolower(cfg$project_report_name), tolower(cfg$language_select), "template", sep = "_") + template_dir <- file.path(template_path, template_dir_name) + + logger::log_info("Creating interactive report.") + create_interactive_report( + template_dir = template_dir, + output_dir = file.path(cfg$output_dir, "report"), + survey_dir = cfg$survey_dir, + real_estate_dir = cfg$real_estate_dir, + language_select = cfg$language_select, investor_name = cfg$investor_name, portfolio_name = cfg$portfolio_name, peer_group = cfg$peer_group, - total_portfolio = total_portfolio, - scenario_selected = "1.5C-Unif", + start_year = cfg$start_year, + select_scenario = cfg$select_scenario, + select_scenario_other = cfg$scenario_other, + portfolio_allocation_method = cfg$portfolio_allocation_method, + scenario_geography = cfg$scenario_geography, + pacta_sectors = cfg$sector_list, + green_techs = cfg$green_techs, + tech_roadmap_sectors = cfg$tech_roadmap_sectors, + pacta_sectors_not_analysed = cfg$pacta_sectors_not_analysed, + audit_file = audit_file, + emissions = emissions, + portfolio_overview = portfolio_overview, + equity_results_portfolio = equity_results_portfolio, + bonds_results_portfolio = bonds_results_portfolio, + equity_results_company = equity_results_company, + bonds_results_company = bonds_results_company, + equity_results_map = equity_results_map, + bonds_results_map = bonds_results_map, + indices_equity_results_portfolio = indices_equity_results_portfolio, + indices_bonds_results_portfolio = indices_bonds_results_portfolio, + peers_equity_results_portfolio = peers_equity_results_portfolio, + peers_bonds_results_portfolio = peers_bonds_results_portfolio, + peers_equity_results_user = peers_equity_results_user, + peers_bonds_results_user = peers_bonds_results_user, + dataframe_translations = dataframe_translations, + js_translations = js_translations, + display_currency = cfg$display_currency, currency_exchange_value = cfg$currency_exchange_value, - log_dir = cfg$output_dir + header_dictionary = header_dictionary, + sector_order = sector_order, + configs = configs ) -} else { - # this is required for the online tool to know that the process has been completed. - logger::log_debug("No executive summary created.") - invisible(file.copy(blank_pdf(), es_dir)) -} -logger::log_info("Portfolio report finished.") + + # create executive summary ----------------------------------------------------- + logger::log_debug("Preparing to create executive summary.") + + survey_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "survey")) + real_estate_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "real_estate")) + score_card_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "score_card")) + output_dir <- file.path(cfg$output_dir) + es_dir <- file.path(cfg$output_dir, "executive_summary") + if (!dir.exists(es_dir)) { + dir.create(es_dir, showWarnings = FALSE, recursive = TRUE) + } + + logger::log_trace("Defining executive summary template paths.") + exec_summary_template_name <- paste0(cfg$project_code, "_", tolower(cfg$language_select), "_exec_summary") + exec_summary_builtin_template_path <- system.file("extdata", exec_summary_template_name, package = "pacta.executive.summary") + invisible(file.copy(exec_summary_builtin_template_path, cfg$output_dir, recursive = TRUE, copy.mode = FALSE)) + exec_summary_template_path <- file.path(cfg$output_dir, exec_summary_template_name) + + if (dir.exists(exec_summary_template_path) && (cfg$peer_group %in% c("assetmanager", "bank", "insurance", "pensionfund"))) { + logger::log_debug("Preparing data for executive summary.") + data_aggregated_filtered <- + prep_data_executive_summary( + investor_name = cfg$investor_name, + portfolio_name = cfg$portfolio_name, + peer_group = cfg$peer_group, + start_year = cfg$start_year, + scenario_source = "GECO2021", + scenario_selected = "1.5C-Unif", + scenario_geography = "Global", + equity_market = "GlobalMarket", + portfolio_allocation_method_equity = "portfolio_weight", + portfolio_allocation_method_bonds = "portfolio_weight", + green_techs = c( + "RenewablesCap", + "HydroCap", + "NuclearCap", + "Hybrid", + "Electric", + "FuelCell", + "Hybrid_HDV", + "Electric_HDV", + "FuelCell_HDV", + "Electric Arc Furnace" + ), + equity_results_portfolio = equity_results_portfolio, + bonds_results_portfolio = bonds_results_portfolio, + peers_equity_results_aggregated = peers_equity_results_portfolio, + peers_bonds_results_aggregated = peers_bonds_results_portfolio, + peers_equity_results_individual = peers_equity_results_user, + peers_bonds_results_individual = peers_bonds_results_user, + indices_equity_results_portfolio = indices_equity_results_portfolio, + indices_bonds_results_portfolio = indices_bonds_results_portfolio, + audit_file = audit_file, + emissions_portfolio = emissions, + score_card_dir = score_card_dir + ) + + logger::log_trace("Checking for real estate data.") + real_estate_flag <- (length(list.files(real_estate_dir)) > 0) + + logger::log_info("Creating executive summary.") + render_executive_summary( + data = data_aggregated_filtered, + language = cfg$language_select, + output_dir = es_dir, + exec_summary_dir = exec_summary_template_path, + survey_dir = survey_dir, + real_estate_dir = real_estate_dir, + real_estate_flag = real_estate_flag, + score_card_dir = score_card_dir, + file_name = "template.Rmd", + investor_name = cfg$investor_name, + portfolio_name = cfg$portfolio_name, + peer_group = cfg$peer_group, + total_portfolio = total_portfolio, + scenario_selected = "1.5C-Unif", + currency_exchange_value = cfg$currency_exchange_value, + log_dir = cfg$output_dir + ) + } else { + # this is required for the online tool to know that the process has been completed. + logger::log_debug("No executive summary created.") + invisible(file.copy(blank_pdf(), es_dir)) + } + + logger::log_info("Portfolio report finished.") +} diff --git a/run-pacta.sh b/run-pacta.sh index 9b77174..dbcc287 100755 --- a/run-pacta.sh +++ b/run-pacta.sh @@ -3,4 +3,4 @@ # Set permissions so that new files can be deleted/overwritten outside docker umask 000 -Rscript --vanilla /workflow.pacta.report/pacta_03.R "${1}" \ +Rscript --vanilla -e "workflow.pacta.report:::run_pacta_reporting_process()" "${1}" \ From cacf5f59685f214213536724e1fd4782c7ea1a6b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 21:30:09 +0200 Subject: [PATCH 026/132] Add testing configs from workflow.pacta --- tests/config/default_2022Q4.json | 8 ++++++++ tests/config/default_2023Q4.json | 8 ++++++++ tests/config/full_params_2023Q4.json | 15 +++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 tests/config/default_2022Q4.json create mode 100644 tests/config/default_2023Q4.json create mode 100644 tests/config/full_params_2023Q4.json diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json new file mode 100644 index 0000000..dfac979 --- /dev/null +++ b/tests/config/default_2022Q4.json @@ -0,0 +1,8 @@ +{ + "holdingsDate": "2022Q4", + "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", + "parameters": { + "portfolio_files": "default_portfolio.csv", + "inherit": "GENERAL_2022Q4" + } +} diff --git a/tests/config/default_2023Q4.json b/tests/config/default_2023Q4.json new file mode 100644 index 0000000..83e2c06 --- /dev/null +++ b/tests/config/default_2023Q4.json @@ -0,0 +1,8 @@ +{ + "holdingsDate": "2023Q4", + "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", + "parameters": { + "portfolio_files": "default_portfolio.csv", + "inherit": "GENERAL_2023Q4" + } +} diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json new file mode 100644 index 0000000..9786e89 --- /dev/null +++ b/tests/config/full_params_2023Q4.json @@ -0,0 +1,15 @@ +{ + "holdingsDate": "2023Q4", + "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", + "parameters": { + "equity_market_list": ["GlobalMarket", "DevelopedMarket", "EmergingMarket"], + "scenario_geographies_list": ["Global", "GlobalAggregate", "NonOECD", "OECD"], + "sector_list": ["Power", "Automotive", "Oil&Gas", "Coal", "Steel", "Aviation", "Cement"], + "time_horizon": 5, + "start_year": 2023, + "holdingsDate": "2023-12-31", + "scenario_sources_list": ["GECO2023", "ISF2023", "WEO2023"], + "project_code": "GENERAL", + "portfolio_files": "default_portfolio.csv" + } +} From 628a32b89086851be90125d684cfd03affd50365 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 21:33:28 +0200 Subject: [PATCH 027/132] Import `logger` functions --- NAMESPACE | 4 ++++ R/workflow.pacta.report-package.R | 10 ++++++++++ 2 files changed, 14 insertions(+) create mode 100644 R/workflow.pacta.report-package.R diff --git a/NAMESPACE b/NAMESPACE index 6ae9268..6a25103 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,2 +1,6 @@ # Generated by roxygen2: do not edit by hand +importFrom(logger,log_fatal) +importFrom(logger,log_info) +importFrom(logger,log_trace) +importFrom(logger,log_warn) diff --git a/R/workflow.pacta.report-package.R b/R/workflow.pacta.report-package.R new file mode 100644 index 0000000..c603273 --- /dev/null +++ b/R/workflow.pacta.report-package.R @@ -0,0 +1,10 @@ +#' @keywords internal +"_PACKAGE" + +## usethis namespace: start +#' @importFrom logger log_fatal +#' @importFrom logger log_info +#' @importFrom logger log_trace +#' @importFrom logger log_warn +## usethis namespace: end +NULL From 1b1da7a0259a38bc47e59988931ed9ad848c6fe5 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 21:35:49 +0200 Subject: [PATCH 028/132] Add run_pacta script --- inst/extdata/scripts/run_pacta_report.R | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 inst/extdata/scripts/run_pacta_report.R diff --git a/inst/extdata/scripts/run_pacta_report.R b/inst/extdata/scripts/run_pacta_report.R new file mode 100644 index 0000000..62335dc --- /dev/null +++ b/inst/extdata/scripts/run_pacta_report.R @@ -0,0 +1,2 @@ +logger::log_threshold(Sys.getenv("LOG_LEVEL", "WARN")) +workflow.pacta.report:::run_pacta_reporting_process(commandArgs(trailingOnly = TRUE)) From ce1257b7b727abb03470eacdac315bc4e9545eb1 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 21:39:10 +0200 Subject: [PATCH 029/132] Call new script from Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 386c4fc..d94a340 100644 --- a/Dockerfile +++ b/Dockerfile @@ -70,4 +70,4 @@ COPY . /workflow.pacta.report/ RUN Rscript -e "pak::local_install(root = '/workflow.pacta.report')" # set default run behavior -ENTRYPOINT ["/workflow.pacta.report/run-pacta.sh"] +ENTRYPOINT ["Rscript", "--vanilla", "/workflow.pacta.report/inst/extdata/scripts/run_pacta_report.R"] From cdfba5e8111ce3794bcf5a7013ec519ac5a719b4 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 21:59:43 +0200 Subject: [PATCH 030/132] Create dir before `chmod`ding --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c7e74bb..a90d6b2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,6 +80,7 @@ jobs: WORKSPACE: ${{ github.workspace }} PARAMETERS: ${{ steps.prepare.outputs.parameters }} run: | + mkdir -p "${WORKSPACE}/${OUTPUT_DIR}" chmod -R 777 "${WORKSPACE}/${OUTPUT_DIR}" docker run \ --network none \ From 88b11757dac391fa2b03fbea613d3be7c0382a86 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:04:12 +0200 Subject: [PATCH 031/132] Update Rbuildignore and dockerignore --- .Rbuildignore | 10 ++++++++++ .dockerignore | 11 +++++++++++ 2 files changed, 21 insertions(+) create mode 100644 .dockerignore diff --git a/.Rbuildignore b/.Rbuildignore index 5163d0b..a82962f 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -1 +1,11 @@ +^.dockerignore$ +^.github/ +^.lintr$ +^Dockerfile$ ^LICENSE\.md$ +^docker-compose.yml$ +^input_dir/ +^output_dir/ +^portfolios/ +^run-pacta.sh$ +^workflow.pacta.Rproj$ diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..70fe075 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +**/.git +**/.github +**/.gitignore +**/Dockerfile +**/*.md +**/*.Rproj +output_dir/**/* +output_dir +pacta-data/**/* +pacta-data +docker-compose.yml From 45a58a88b1a72f8d4b403185fb4086ccab6445be Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:07:54 +0200 Subject: [PATCH 032/132] Use import logging functions from package namespace --- R/run_pacta_reporting_process.R | 82 ++++++++++++++++----------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index b21c6a1..206fee8 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -15,27 +15,27 @@ run_pacta_reporting_process <- function() { # ------------------------------------------------------------------------- - logger::log_info("Starting portfolio report process") + log_info("Starting portfolio report process") - logger::log_trace("Determining configuration file path") + log_trace("Determining configuration file path") cfg_path <- commandArgs(trailingOnly = TRUE) if (length(cfg_path) == 0 || cfg_path == "") { - logger::log_warn("No configuration file specified, using default") + log_warn("No configuration file specified, using default") cfg_path <- "/workflow.pacta.report/input_dir/default_config.json" } - logger::log_debug("Loading configuration from file: \"{cfg_path}\".") + log_debug("Loading configuration from file: \"{cfg_path}\".") cfg <- fromJSON(cfg_path) # quit if there's no relevant PACTA assets ------------------------------------- - logger::log_debug("Checking for PACTA relevant data in portfolio results.") + log_debug("Checking for PACTA relevant data in portfolio results.") total_portfolio_path <- file.path(cfg$output_dir, "total_portfolio.rds") if (file.exists(total_portfolio_path)) { total_portfolio <- readRDS(total_portfolio_path) - logger::log_trace("Checking for PACTA relevant data in file: \"{total_portfolio_path}\".") + log_trace("Checking for PACTA relevant data in file: \"{total_portfolio_path}\".") quit_if_no_pacta_relevant_data(total_portfolio) } else { - logger::log_warn("file \"{total_portfolio_path}\" does not exist.") + log_warn("file \"{total_portfolio_path}\" does not exist.") warning("This is weird... the `total_portfolio.rds` file does not exist in the `30_Processed_inputs` directory.") } @@ -43,16 +43,16 @@ run_pacta_reporting_process <- function() { # fix parameters --------------------------------------------------------------- if (cfg$project_code == "GENERAL") { - logger::log_warn("Overriding language selection to \"EN\" for \"GENERAL\".") + log_warn("Overriding language selection to \"EN\" for \"GENERAL\".") cfg$language_select <- "EN" } else { - logger::log_trace("Using language selection: \"{cfg$language_select}\".") + log_trace("Using language selection: \"{cfg$language_select}\".") } # load PACTA results ----------------------------------------------------------- - logger::log_info("Loading PACTA results.") + log_info("Loading PACTA results.") readRDS_or_return_alt_data <- function(filepath, alt_return = NULL) { if (file.exists(filepath)) { @@ -77,139 +77,139 @@ run_pacta_reporting_process <- function() { data } - logger::log_debug("Loading audit file.") + log_debug("Loading audit file.") audit_file <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "audit_file.rds"), alt_return = empty_audit_file() ) audit_file <- add_inv_and_port_names_if_needed(audit_file) - logger::log_debug("Loading portfolio overview.") + log_debug("Loading portfolio overview.") portfolio_overview <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "overview_portfolio.rds"), alt_return = empty_portfolio_overview() ) portfolio_overview <- add_inv_and_port_names_if_needed(portfolio_overview) - logger::log_debug("Loading emissions.") + log_debug("Loading emissions.") emissions <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "emissions.rds"), alt_return = empty_emissions_results() ) emissions <- add_inv_and_port_names_if_needed(emissions) - logger::log_debug("Loading total portfolio results.") + log_debug("Loading total portfolio results.") total_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "total_portfolio.rds"), alt_return = empty_portfolio_results() ) total_portfolio <- add_inv_and_port_names_if_needed(total_portfolio) - logger::log_debug("Loading portfolio equity results.") + log_debug("Loading portfolio equity results.") equity_results_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_portfolio.rds"), alt_return = empty_portfolio_results() ) equity_results_portfolio <- add_inv_and_port_names_if_needed(equity_results_portfolio) - logger::log_debug("Loading portfolio bonds results.") + log_debug("Loading portfolio bonds results.") bonds_results_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_portfolio.rds"), alt_return = empty_portfolio_results() ) bonds_results_portfolio <- add_inv_and_port_names_if_needed(bonds_results_portfolio) - logger::log_debug("Loading company equity results.") + log_debug("Loading company equity results.") equity_results_company <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_company.rds"), alt_return = empty_company_results() ) equity_results_company <- add_inv_and_port_names_if_needed(equity_results_company) - logger::log_debug("Loading company bonds results.") + log_debug("Loading company bonds results.") bonds_results_company <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_company.rds"), alt_return = empty_company_results() ) bonds_results_company <- add_inv_and_port_names_if_needed(bonds_results_company) - logger::log_debug("Loading equity map results.") + log_debug("Loading equity map results.") equity_results_map <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_map.rds"), alt_return = empty_map_results() ) equity_results_map <- add_inv_and_port_names_if_needed(equity_results_map) - logger::log_debug("Loading bonds map results.") + log_debug("Loading bonds map results.") bonds_results_map <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_map.rds"), alt_return = empty_map_results() ) bonds_results_map <- add_inv_and_port_names_if_needed(bonds_results_map) - logger::log_debug("Loading portfolio equity peer results.") + log_debug("Loading portfolio equity peer results.") peers_equity_results_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio.rds")), alt_return = empty_portfolio_results() ) - logger::log_debug("Loading portfolio bonds peer results.") + log_debug("Loading portfolio bonds peer results.") peers_bonds_results_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio.rds")), alt_return = empty_portfolio_results() ) - logger::log_debug("Loading index equity peer results.") + log_debug("Loading index equity peer results.") peers_equity_results_user <- readRDS_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio_ind.rds")), alt_return = empty_portfolio_results() ) - logger::log_debug("Loading index bonds peer results.") + log_debug("Loading index bonds peer results.") peers_bonds_results_user <- readRDS_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio_ind.rds")), alt_return = empty_portfolio_results() ) - logger::log_debug("Loading index equity portfolio results.") + log_debug("Loading index equity portfolio results.") indices_equity_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_equity_results_portfolio.rds")) - logger::log_debug("Loading index bonds portfolio results.") + log_debug("Loading index bonds portfolio results.") indices_bonds_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_bonds_results_portfolio.rds")) # create interactive report ---------------------------------------------------- - logger::log_debug("Preparing to create interactive report.") + log_debug("Preparing to create interactive report.") survey_dir <- file.path(cfg$user_results_path, cfg$project_code, "survey") real_estate_dir <- file.path(cfg$user_results_path, cfg$project_code, "real_estate") output_dir <- file.path(cfg$output_dir) - logger::log_debug("Loading data frame label translations.") + log_debug("Loading data frame label translations.") dataframe_translations <- readr::read_csv( system.file("extdata/translation/dataframe_labels.csv", package = "pacta.portfolio.report"), col_types = cols() ) - logger::log_debug("Loading data frame header translations.") + log_debug("Loading data frame header translations.") header_dictionary <- readr::read_csv( system.file("extdata/translation/dataframe_headers.csv", package = "pacta.portfolio.report"), col_types = cols() ) - logger::log_debug("Loading JavaScript label translations.") + log_debug("Loading JavaScript label translations.") js_translations <- jsonlite::fromJSON( txt = system.file("extdata/translation/js_labels.json", package = "pacta.portfolio.report") ) - logger::log_debug("Loading sector order.") + log_debug("Loading sector order.") sector_order <- readr::read_csv( system.file("extdata/sector_order/sector_order.csv", package = "pacta.portfolio.report"), col_types = cols() ) # combine config files to send to create_interactive_report() - logger::log_trace("Defining configs and manifest.") + log_trace("Defining configs and manifest.") pacta_data_public_manifest <- list( creation_time_date = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$creation_time_date, @@ -225,12 +225,12 @@ run_pacta_reporting_process <- function() { # workaround a bug in {config} v0.3.2 that only adds "config" class to objects it creates class(configs$portfolio_config) <- c(class(configs$portfolio_config), "list") - logger::log_trace("Defining interactive report template paths.") + log_trace("Defining interactive report template paths.") template_path <- system.file("templates", package = "pacta.portfolio.report") #TODO: generalize this to accept non-builtin templates template_dir_name <- paste(tolower(cfg$project_report_name), tolower(cfg$language_select), "template", sep = "_") template_dir <- file.path(template_path, template_dir_name) - logger::log_info("Creating interactive report.") + log_info("Creating interactive report.") create_interactive_report( template_dir = template_dir, output_dir = file.path(cfg$output_dir, "report"), @@ -275,7 +275,7 @@ run_pacta_reporting_process <- function() { # create executive summary ----------------------------------------------------- - logger::log_debug("Preparing to create executive summary.") + log_debug("Preparing to create executive summary.") survey_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "survey")) real_estate_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "real_estate")) @@ -286,14 +286,14 @@ run_pacta_reporting_process <- function() { dir.create(es_dir, showWarnings = FALSE, recursive = TRUE) } - logger::log_trace("Defining executive summary template paths.") + log_trace("Defining executive summary template paths.") exec_summary_template_name <- paste0(cfg$project_code, "_", tolower(cfg$language_select), "_exec_summary") exec_summary_builtin_template_path <- system.file("extdata", exec_summary_template_name, package = "pacta.executive.summary") invisible(file.copy(exec_summary_builtin_template_path, cfg$output_dir, recursive = TRUE, copy.mode = FALSE)) exec_summary_template_path <- file.path(cfg$output_dir, exec_summary_template_name) if (dir.exists(exec_summary_template_path) && (cfg$peer_group %in% c("assetmanager", "bank", "insurance", "pensionfund"))) { - logger::log_debug("Preparing data for executive summary.") + log_debug("Preparing data for executive summary.") data_aggregated_filtered <- prep_data_executive_summary( investor_name = cfg$investor_name, @@ -331,10 +331,10 @@ run_pacta_reporting_process <- function() { score_card_dir = score_card_dir ) - logger::log_trace("Checking for real estate data.") + log_trace("Checking for real estate data.") real_estate_flag <- (length(list.files(real_estate_dir)) > 0) - logger::log_info("Creating executive summary.") + log_info("Creating executive summary.") render_executive_summary( data = data_aggregated_filtered, language = cfg$language_select, @@ -355,9 +355,9 @@ run_pacta_reporting_process <- function() { ) } else { # this is required for the online tool to know that the process has been completed. - logger::log_debug("No executive summary created.") + log_debug("No executive summary created.") invisible(file.copy(blank_pdf(), es_dir)) } - logger::log_info("Portfolio report finished.") + log_info("Portfolio report finished.") } From 10415699c42b7fb35dc114ac6436d8fdcb64c8c5 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:10:03 +0200 Subject: [PATCH 033/132] Update upload URL --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a90d6b2..5fb2cd4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ on: results-url: description: azure blob store path for results required: false - default: "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results" + default: "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-report-results" type: string jobs: From b5bccb39dfddb5362f7e9e6b93483fec924809d6 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:14:32 +0200 Subject: [PATCH 034/132] Use default testing portfolio for analysis results --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5fb2cd4..ef37ff2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,6 @@ jobs: - name: Run PACTA Analysis env: FULL_IMAGE_NAME: ${{ inputs.full-image-name }} - PORTFOLIO_DIR: portfolios OUTPUT_DIR: output_dir PACTA_DATA_DIR: pacta-data WORKSPACE: ${{ github.workspace }} @@ -85,9 +84,9 @@ jobs: docker run \ --network none \ --env LOG_LEVEL=DEBUG \ + --env PORTFOLIO_DIR=/workflow.pacta/portfolios/ --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ --mount type=bind,source=${WORKSPACE}/${OUTPUT_DIR},target=/mnt/output_dir \ - --mount type=bind,source=${WORKSPACE}/${PORTFOLIO_DIR},target=/mnt/portfolios \ ghcr.io/rmi-pacta/workflow.pacta:main \ $PARAMETERS From 2b9a7ba20cbfa1c9a4b7f6c37539f750d4f596c0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:18:33 +0200 Subject: [PATCH 035/132] fix line continuation --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef37ff2..38b7ccb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -84,7 +84,7 @@ jobs: docker run \ --network none \ --env LOG_LEVEL=DEBUG \ - --env PORTFOLIO_DIR=/workflow.pacta/portfolios/ + --env PORTFOLIO_DIR=/workflow.pacta/portfolios/ \ --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ --mount type=bind,source=${WORKSPACE}/${OUTPUT_DIR},target=/mnt/output_dir \ ghcr.io/rmi-pacta/workflow.pacta:main \ From f1dd139e63e43714fe2602acc8e457282820c96c Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:22:28 +0200 Subject: [PATCH 036/132] Factor utility functions to separate file --- R/run_pacta_reporting_process.R | 23 ----------------------- R/utils.R | 22 ++++++++++++++++++++++ 2 files changed, 22 insertions(+), 23 deletions(-) create mode 100644 R/utils.R diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 206fee8..fed3a1c 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -54,29 +54,6 @@ run_pacta_reporting_process <- function() { log_info("Loading PACTA results.") - readRDS_or_return_alt_data <- function(filepath, alt_return = NULL) { - if (file.exists(filepath)) { - return(readRDS(filepath)) - } - alt_return - } - - add_inv_and_port_names_if_needed <- function(data) { - if (!inherits(data, "data.frame")) { - return(data) - } - - if (!"portfolio_name" %in% names(data)) { - data <- mutate(data, portfolio_name = cfg$portfolio_name, .before = everything()) - } - - if (!"investor_name" %in% names(data)) { - data <- mutate(data, investor_name = cfg$investor_name, .before = everything()) - } - - data - } - log_debug("Loading audit file.") audit_file <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "audit_file.rds"), diff --git a/R/utils.R b/R/utils.R new file mode 100644 index 0000000..21062d8 --- /dev/null +++ b/R/utils.R @@ -0,0 +1,22 @@ +readRDS_or_return_alt_data <- function(filepath, alt_return = NULL) { + if (file.exists(filepath)) { + return(readRDS(filepath)) + } + alt_return +} + +add_inv_and_port_names_if_needed <- function(data) { + if (!inherits(data, "data.frame")) { + return(data) + } + + if (!"portfolio_name" %in% names(data)) { + data <- mutate(data, portfolio_name = cfg$portfolio_name, .before = everything()) + } + + if (!"investor_name" %in% names(data)) { + data <- mutate(data, investor_name = cfg$investor_name, .before = everything()) + } + + data +} From a71202b10ad72328fc2e4f62e8fb642a7e91d004 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:26:05 +0200 Subject: [PATCH 037/132] Remove TODO https://github.com/RMI-PACTA/workflow.pacta.report/issues/19 --- R/run_pacta_reporting_process.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index fed3a1c..daeeab6 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -203,7 +203,7 @@ run_pacta_reporting_process <- function() { class(configs$portfolio_config) <- c(class(configs$portfolio_config), "list") log_trace("Defining interactive report template paths.") - template_path <- system.file("templates", package = "pacta.portfolio.report") #TODO: generalize this to accept non-builtin templates + template_path <- system.file("templates", package = "pacta.portfolio.report") template_dir_name <- paste(tolower(cfg$project_report_name), tolower(cfg$language_select), "template", sep = "_") template_dir <- file.path(template_path, template_dir_name) From 9d44006d58219097cc2a0412f3e358318797904b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:27:36 +0200 Subject: [PATCH 038/132] linting --- R/run_pacta_reporting_process.R | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index daeeab6..2e6e8f8 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -265,11 +265,18 @@ run_pacta_reporting_process <- function() { log_trace("Defining executive summary template paths.") exec_summary_template_name <- paste0(cfg$project_code, "_", tolower(cfg$language_select), "_exec_summary") - exec_summary_builtin_template_path <- system.file("extdata", exec_summary_template_name, package = "pacta.executive.summary") + exec_summary_builtin_template_path <- system.file( + "extdata", exec_summary_template_name, + package = "pacta.executive.summary" + ) invisible(file.copy(exec_summary_builtin_template_path, cfg$output_dir, recursive = TRUE, copy.mode = FALSE)) exec_summary_template_path <- file.path(cfg$output_dir, exec_summary_template_name) - if (dir.exists(exec_summary_template_path) && (cfg$peer_group %in% c("assetmanager", "bank", "insurance", "pensionfund"))) { + if ( + dir.exists(exec_summary_template_path) && ( + cfg$peer_group %in% c("assetmanager", "bank", "insurance", "pensionfund") + ) + ) { log_debug("Preparing data for executive summary.") data_aggregated_filtered <- prep_data_executive_summary( @@ -294,7 +301,7 @@ run_pacta_reporting_process <- function() { "Electric_HDV", "FuelCell_HDV", "Electric Arc Furnace" - ), + ), equity_results_portfolio = equity_results_portfolio, bonds_results_portfolio = bonds_results_portfolio, peers_equity_results_aggregated = peers_equity_results_portfolio, From 0d510c098987b0ff1488584fe15dc128f38f9338 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:32:17 +0200 Subject: [PATCH 039/132] Remove second `chmod` --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 38b7ccb..7fc31c4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -99,7 +99,6 @@ jobs: WORKSPACE: ${{ github.workspace }} PARAMETERS: ${{ steps.prepare.outputs.parameters }} run: | - chmod -R 777 "${WORKSPACE}/${OUTPUT_DIR}" docker run \ --network none \ --env LOG_LEVEL=DEBUG \ From 3dadeb23bdb6061d284a313b5220aaf06eeaa3ac Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:33:56 +0200 Subject: [PATCH 040/132] render Docs --- man/workflow.pacta.report-package.Rd | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 man/workflow.pacta.report-package.Rd diff --git a/man/workflow.pacta.report-package.Rd b/man/workflow.pacta.report-package.Rd new file mode 100644 index 0000000..909c19e --- /dev/null +++ b/man/workflow.pacta.report-package.Rd @@ -0,0 +1,25 @@ +% Generated by roxygen2: do not edit by hand +% Please edit documentation in R/workflow.pacta.report-package.R +\docType{package} +\name{workflow.pacta.report-package} +\alias{workflow.pacta.report} +\alias{workflow.pacta.report-package} +\title{workflow.pacta.report: Reporting functionality for PACTA} +\description{ +Create Interactive Reports and Executive Summaries for PACTA +} +\author{ +\strong{Maintainer}: Alex Axthelm \email{aaxthelm@rmi.org} (\href{https://orcid.org/0000-0001-8579-8565}{ORCID}) [contractor] + +Authors: +\itemize{ + \item Jackson Hoffart \email{jackson.hoffart@gmail.com} (\href{https://orcid.org/0000-0002-8600-5042}{ORCID}) [contractor] +} + +Other contributors: +\itemize{ + \item RMI \email{PACTA4investors@rmi.org} [copyright holder, funder] +} + +} +\keyword{internal} From f558892e3216eef0d337d6814812fd61a5bee602 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:37:22 +0200 Subject: [PATCH 041/132] import log_debug --- NAMESPACE | 1 + R/workflow.pacta.report-package.R | 1 + 2 files changed, 2 insertions(+) diff --git a/NAMESPACE b/NAMESPACE index 6a25103..38ac726 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,5 +1,6 @@ # Generated by roxygen2: do not edit by hand +importFrom(logger,log_debug) importFrom(logger,log_fatal) importFrom(logger,log_info) importFrom(logger,log_trace) diff --git a/R/workflow.pacta.report-package.R b/R/workflow.pacta.report-package.R index c603273..a270edb 100644 --- a/R/workflow.pacta.report-package.R +++ b/R/workflow.pacta.report-package.R @@ -2,6 +2,7 @@ "_PACKAGE" ## usethis namespace: start +#' @importFrom logger log_debug #' @importFrom logger log_fatal #' @importFrom logger log_info #' @importFrom logger log_trace From ae8abf04457372654783ab7dadf5033fa72cf6ce Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:38:32 +0200 Subject: [PATCH 042/132] Add R checks from RMI-PACTA/actions --- .github/workflows/R.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/R.yml diff --git a/.github/workflows/R.yml b/.github/workflows/R.yml new file mode 100644 index 0000000..1023ad1 --- /dev/null +++ b/.github/workflows/R.yml @@ -0,0 +1,22 @@ +--- +# This example file will enable R language checks on push or PR to the main +# branch. +# It will also run the checks every weeknight at midnight UTC +# +# Note the @main in `uses:` on the last line. This will call the latest version +# of the workflow from the `main` brnach in the RMI-PACTA/actions repo. You can +# also specify a tag from that repo, or a commit SHA to pin action versions. +on: + pull_request: + push: + branches: [main] + schedule: + - cron: '0 0 * * 1,2,3,4,5' + workflow_dispatch: + +name: R + +jobs: + R-package: + name: R Package Checks + uses: RMI-PACTA/actions/.github/workflows/R.yml@main From 24823398d2b471ca64ec4c78768a899ca7387bca Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:40:59 +0200 Subject: [PATCH 043/132] Remove unused variables --- R/run_pacta_reporting_process.R | 2 -- 1 file changed, 2 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 2e6e8f8..0e4ffbe 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -160,7 +160,6 @@ run_pacta_reporting_process <- function() { survey_dir <- file.path(cfg$user_results_path, cfg$project_code, "survey") real_estate_dir <- file.path(cfg$user_results_path, cfg$project_code, "real_estate") - output_dir <- file.path(cfg$output_dir) log_debug("Loading data frame label translations.") dataframe_translations <- readr::read_csv( @@ -257,7 +256,6 @@ run_pacta_reporting_process <- function() { survey_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "survey")) real_estate_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "real_estate")) score_card_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "score_card")) - output_dir <- file.path(cfg$output_dir) es_dir <- file.path(cfg$output_dir, "executive_summary") if (!dir.exists(es_dir)) { dir.create(es_dir, showWarnings = FALSE, recursive = TRUE) From 3adcfc8b87775fc1cea788a1a14e9dfc432d2baf Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:44:02 +0200 Subject: [PATCH 044/132] Report image doesn't need a portfolio mount --- .github/workflows/test.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7fc31c4..07cb33f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -93,7 +93,6 @@ jobs: - name: Run Docker Image env: FULL_IMAGE_NAME: ${{ inputs.full-image-name }} - PORTFOLIO_DIR: portfolios OUTPUT_DIR: output_dir PACTA_DATA_DIR: pacta-data WORKSPACE: ${{ github.workspace }} @@ -104,7 +103,6 @@ jobs: --env LOG_LEVEL=DEBUG \ --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ --mount type=bind,source=${WORKSPACE}/${OUTPUT_DIR},target=/mnt/output_dir \ - --mount type=bind,source=${WORKSPACE}/${PORTFOLIO_DIR},target=/mnt/portfolios \ $FULL_IMAGE_NAME # $FULL_IMAGE_NAME \ # $PARAMETERS From 835c0870374d95210af0072bf24f55808370b189 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:45:42 +0200 Subject: [PATCH 045/132] Remove unused shell script --- run-pacta.sh | 6 ------ 1 file changed, 6 deletions(-) delete mode 100755 run-pacta.sh diff --git a/run-pacta.sh b/run-pacta.sh deleted file mode 100755 index dbcc287..0000000 --- a/run-pacta.sh +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/bash - -# Set permissions so that new files can be deleted/overwritten outside docker -umask 000 - -Rscript --vanilla -e "workflow.pacta.report:::run_pacta_reporting_process()" "${1}" \ From e9685b3dc8bc2c4100d82e4d7c3922662e83d1c9 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:53:43 +0200 Subject: [PATCH 046/132] Namespace functions from `pacta.portfolio.utils` --- R/run_pacta_reporting_process.R | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 0e4ffbe..f8560cf 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,6 +1,5 @@ run_pacta_reporting_process <- function() { suppressPackageStartupMessages({ - library(pacta.portfolio.utils) library(pacta.portfolio.report) library(pacta.executive.summary) library(dplyr) @@ -33,7 +32,7 @@ run_pacta_reporting_process <- function() { if (file.exists(total_portfolio_path)) { total_portfolio <- readRDS(total_portfolio_path) log_trace("Checking for PACTA relevant data in file: \"{total_portfolio_path}\".") - quit_if_no_pacta_relevant_data(total_portfolio) + pacta.portfolio.utils::quit_if_no_pacta_relevant_data(total_portfolio) } else { log_warn("file \"{total_portfolio_path}\" does not exist.") warning("This is weird... the `total_portfolio.rds` file does not exist in the `30_Processed_inputs` directory.") @@ -57,95 +56,95 @@ run_pacta_reporting_process <- function() { log_debug("Loading audit file.") audit_file <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "audit_file.rds"), - alt_return = empty_audit_file() + alt_return = pacta.portfolio.utils::empty_audit_file() ) audit_file <- add_inv_and_port_names_if_needed(audit_file) log_debug("Loading portfolio overview.") portfolio_overview <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "overview_portfolio.rds"), - alt_return = empty_portfolio_overview() + alt_return = pacta.portfolio.utils::empty_portfolio_overview() ) portfolio_overview <- add_inv_and_port_names_if_needed(portfolio_overview) log_debug("Loading emissions.") emissions <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "emissions.rds"), - alt_return = empty_emissions_results() + alt_return = pacta.portfolio.utils::empty_emissions_results() ) emissions <- add_inv_and_port_names_if_needed(emissions) log_debug("Loading total portfolio results.") total_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "total_portfolio.rds"), - alt_return = empty_portfolio_results() + alt_return = pacta.portfolio.utils::empty_portfolio_results() ) total_portfolio <- add_inv_and_port_names_if_needed(total_portfolio) log_debug("Loading portfolio equity results.") equity_results_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_portfolio.rds"), - alt_return = empty_portfolio_results() + alt_return = pacta.portfolio.utils::empty_portfolio_results() ) equity_results_portfolio <- add_inv_and_port_names_if_needed(equity_results_portfolio) log_debug("Loading portfolio bonds results.") bonds_results_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_portfolio.rds"), - alt_return = empty_portfolio_results() + alt_return = pacta.portfolio.utils::empty_portfolio_results() ) bonds_results_portfolio <- add_inv_and_port_names_if_needed(bonds_results_portfolio) log_debug("Loading company equity results.") equity_results_company <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_company.rds"), - alt_return = empty_company_results() + alt_return = pacta.portfolio.utils::empty_company_results() ) equity_results_company <- add_inv_and_port_names_if_needed(equity_results_company) log_debug("Loading company bonds results.") bonds_results_company <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_company.rds"), - alt_return = empty_company_results() + alt_return = pacta.portfolio.utils::empty_company_results() ) bonds_results_company <- add_inv_and_port_names_if_needed(bonds_results_company) log_debug("Loading equity map results.") equity_results_map <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_map.rds"), - alt_return = empty_map_results() + alt_return = pacta.portfolio.utils::empty_map_results() ) equity_results_map <- add_inv_and_port_names_if_needed(equity_results_map) log_debug("Loading bonds map results.") bonds_results_map <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_map.rds"), - alt_return = empty_map_results() + alt_return = pacta.portfolio.utils::empty_map_results() ) bonds_results_map <- add_inv_and_port_names_if_needed(bonds_results_map) log_debug("Loading portfolio equity peer results.") peers_equity_results_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio.rds")), - alt_return = empty_portfolio_results() + alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading portfolio bonds peer results.") peers_bonds_results_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio.rds")), - alt_return = empty_portfolio_results() + alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading index equity peer results.") peers_equity_results_user <- readRDS_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio_ind.rds")), - alt_return = empty_portfolio_results() + alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading index bonds peer results.") peers_bonds_results_user <- readRDS_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio_ind.rds")), - alt_return = empty_portfolio_results() + alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading index equity portfolio results.") From 6f16d2f408a70c78fc46a282db022cf78af25e9b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:54:20 +0200 Subject: [PATCH 047/132] Namespace functions from `pacta.portfolio.report` --- R/run_pacta_reporting_process.R | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index f8560cf..fa529bb 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,6 +1,5 @@ run_pacta_reporting_process <- function() { suppressPackageStartupMessages({ - library(pacta.portfolio.report) library(pacta.executive.summary) library(dplyr) library(readr) @@ -206,7 +205,7 @@ run_pacta_reporting_process <- function() { template_dir <- file.path(template_path, template_dir_name) log_info("Creating interactive report.") - create_interactive_report( + pacta.portfolio.report::create_interactive_report( template_dir = template_dir, output_dir = file.path(cfg$output_dir, "report"), survey_dir = cfg$survey_dir, From b3b873c69a8f7d515ec92e31441020cc388fc047 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:55:19 +0200 Subject: [PATCH 048/132] Namespace functions from `pacta.executive.summary` --- R/run_pacta_reporting_process.R | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index fa529bb..5650a38 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,6 +1,5 @@ run_pacta_reporting_process <- function() { suppressPackageStartupMessages({ - library(pacta.executive.summary) library(dplyr) library(readr) library(jsonlite) @@ -275,7 +274,7 @@ run_pacta_reporting_process <- function() { ) { log_debug("Preparing data for executive summary.") data_aggregated_filtered <- - prep_data_executive_summary( + pacta.executive.summary::prep_data_executive_summary( investor_name = cfg$investor_name, portfolio_name = cfg$portfolio_name, peer_group = cfg$peer_group, @@ -315,7 +314,7 @@ run_pacta_reporting_process <- function() { real_estate_flag <- (length(list.files(real_estate_dir)) > 0) log_info("Creating executive summary.") - render_executive_summary( + pacta.executive.summary::render_executive_summary( data = data_aggregated_filtered, language = cfg$language_select, output_dir = es_dir, @@ -336,7 +335,7 @@ run_pacta_reporting_process <- function() { } else { # this is required for the online tool to know that the process has been completed. log_debug("No executive summary created.") - invisible(file.copy(blank_pdf(), es_dir)) + invisible(file.copy(pacta.executive.summary::blank_pdf(), es_dir)) } log_info("Portfolio report finished.") From 96977ac0537a629842143024b122a0ba3881f228 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:55:53 +0200 Subject: [PATCH 049/132] namespace functions from `jsonlite` --- R/run_pacta_reporting_process.R | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 5650a38..76c0d8c 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -2,7 +2,6 @@ run_pacta_reporting_process <- function() { suppressPackageStartupMessages({ library(dplyr) library(readr) - library(jsonlite) library(fs) }) @@ -21,7 +20,7 @@ run_pacta_reporting_process <- function() { cfg_path <- "/workflow.pacta.report/input_dir/default_config.json" } log_debug("Loading configuration from file: \"{cfg_path}\".") - cfg <- fromJSON(cfg_path) + cfg <- jsonlite::fromJSON(cfg_path) # quit if there's no relevant PACTA assets ------------------------------------- From a6a9d82b1cee4c3c666772b46d39823d37213c27 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:56:22 +0200 Subject: [PATCH 050/132] Remvoe library call to `fs` and `dplyr` --- R/run_pacta_reporting_process.R | 2 -- 1 file changed, 2 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 76c0d8c..8979e1f 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,8 +1,6 @@ run_pacta_reporting_process <- function() { suppressPackageStartupMessages({ - library(dplyr) library(readr) - library(fs) }) # defaulting to WARN to maintain current (silent) behavior. From ea3caab5d884b21dbcb2a29f2666017b00ebdb33 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:57:51 +0200 Subject: [PATCH 051/132] Namespace functions from `readr` --- R/run_pacta_reporting_process.R | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 8979e1f..58cc4ca 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,7 +1,4 @@ run_pacta_reporting_process <- function() { - suppressPackageStartupMessages({ - library(readr) - }) # defaulting to WARN to maintain current (silent) behavior. logger::log_threshold(Sys.getenv("LOG_LEVEL", "WARN")) @@ -158,13 +155,13 @@ run_pacta_reporting_process <- function() { log_debug("Loading data frame label translations.") dataframe_translations <- readr::read_csv( system.file("extdata/translation/dataframe_labels.csv", package = "pacta.portfolio.report"), - col_types = cols() + col_types = readr::cols() ) log_debug("Loading data frame header translations.") header_dictionary <- readr::read_csv( system.file("extdata/translation/dataframe_headers.csv", package = "pacta.portfolio.report"), - col_types = cols() + col_types = readr::cols() ) log_debug("Loading JavaScript label translations.") @@ -175,7 +172,7 @@ run_pacta_reporting_process <- function() { log_debug("Loading sector order.") sector_order <- readr::read_csv( system.file("extdata/sector_order/sector_order.csv", package = "pacta.portfolio.report"), - col_types = cols() + col_types = readr::cols() ) # combine config files to send to create_interactive_report() From 39052da92776794a88d84302e27a12b240969018 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 22:59:11 +0200 Subject: [PATCH 052/132] Namespace functions in utils --- R/utils.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/utils.R b/R/utils.R index 21062d8..2a1c5b5 100644 --- a/R/utils.R +++ b/R/utils.R @@ -11,11 +11,11 @@ add_inv_and_port_names_if_needed <- function(data) { } if (!"portfolio_name" %in% names(data)) { - data <- mutate(data, portfolio_name = cfg$portfolio_name, .before = everything()) + data <- dplyr::mutate(data, portfolio_name = cfg$portfolio_name, .before = dplyr::everything()) } if (!"investor_name" %in% names(data)) { - data <- mutate(data, investor_name = cfg$investor_name, .before = everything()) + data <- dplyr::mutate(data, investor_name = cfg$investor_name, .before = dplyr::everything()) } data From 47ee5970aaef2dc1dd347821851c1e75db108546 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 23:06:07 +0200 Subject: [PATCH 053/132] Pass arguments to utility functions explicitly Previously, had been inheriting from parent env. --- R/run_pacta_reporting_process.R | 60 +++++++++++++++++++++++++++------ R/utils.R | 6 ++-- 2 files changed, 53 insertions(+), 13 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 58cc4ca..6f9f569 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -50,70 +50,110 @@ run_pacta_reporting_process <- function() { filepath = file.path(cfg$output_dir, "audit_file.rds"), alt_return = pacta.portfolio.utils::empty_audit_file() ) - audit_file <- add_inv_and_port_names_if_needed(audit_file) + audit_file <- add_inv_and_port_names_if_needed( + data = audit_file, + portfolio_name = cfg[["portfolio_name"]], + investor_name = cfg[["investor_name"]] + ) log_debug("Loading portfolio overview.") portfolio_overview <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "overview_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_overview() ) - portfolio_overview <- add_inv_and_port_names_if_needed(portfolio_overview) + portfolio_overview <- add_inv_and_port_names_if_needed( + data = portfolio_overview, + portfolio_name = cfg[["portfolio_name"]], + investor_name = cfg[["investor_name"]] + ) log_debug("Loading emissions.") emissions <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "emissions.rds"), alt_return = pacta.portfolio.utils::empty_emissions_results() ) - emissions <- add_inv_and_port_names_if_needed(emissions) + emissions <- add_inv_and_port_names_if_needed( + data = emissions, + portfolio_name = cfg[["portfolio_name"]], + investor_name = cfg[["investor_name"]] + ) log_debug("Loading total portfolio results.") total_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "total_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) - total_portfolio <- add_inv_and_port_names_if_needed(total_portfolio) + total_portfolio <- add_inv_and_port_names_if_needed( + data = total_portfolio, + portfolio_name = cfg[["portfolio_name"]], + investor_name = cfg[["investor_name"]] + ) log_debug("Loading portfolio equity results.") equity_results_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) - equity_results_portfolio <- add_inv_and_port_names_if_needed(equity_results_portfolio) + equity_results_portfolio <- add_inv_and_port_names_if_needed( + data = equity_results_portfolio, + portfolio_name = cfg[["portfolio_name"]], + investor_name = cfg[["investor_name"]] + ) log_debug("Loading portfolio bonds results.") bonds_results_portfolio <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) - bonds_results_portfolio <- add_inv_and_port_names_if_needed(bonds_results_portfolio) + bonds_results_portfolio <- add_inv_and_port_names_if_needed( + data = bonds_results_portfolio, + portfolio_name = cfg[["portfolio_name"]], + investor_name = cfg[["investor_name"]] + ) log_debug("Loading company equity results.") equity_results_company <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_company.rds"), alt_return = pacta.portfolio.utils::empty_company_results() ) - equity_results_company <- add_inv_and_port_names_if_needed(equity_results_company) + equity_results_company <- add_inv_and_port_names_if_needed( + data = equity_results_company, + portfolio_name = cfg[["portfolio_name"]], + investor_name = cfg[["investor_name"]] + ) log_debug("Loading company bonds results.") bonds_results_company <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_company.rds"), alt_return = pacta.portfolio.utils::empty_company_results() ) - bonds_results_company <- add_inv_and_port_names_if_needed(bonds_results_company) + bonds_results_company <- add_inv_and_port_names_if_needed( + data = bonds_results_company, + portfolio_name = cfg[["portfolio_name"]], + investor_name = cfg[["investor_name"]] + ) log_debug("Loading equity map results.") equity_results_map <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_map.rds"), alt_return = pacta.portfolio.utils::empty_map_results() ) - equity_results_map <- add_inv_and_port_names_if_needed(equity_results_map) + equity_results_map <- add_inv_and_port_names_if_needed( + data = equity_results_map, + portfolio_name = cfg[["portfolio_name"]], + investor_name = cfg[["investor_name"]] + ) log_debug("Loading bonds map results.") bonds_results_map <- readRDS_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_map.rds"), alt_return = pacta.portfolio.utils::empty_map_results() ) - bonds_results_map <- add_inv_and_port_names_if_needed(bonds_results_map) + bonds_results_map <- add_inv_and_port_names_if_needed( + data = bonds_results_map, + portfolio_name = cfg[["portfolio_name"]], + investor_name = cfg[["investor_name"]] + ) log_debug("Loading portfolio equity peer results.") peers_equity_results_portfolio <- readRDS_or_return_alt_data( diff --git a/R/utils.R b/R/utils.R index 2a1c5b5..bee56ba 100644 --- a/R/utils.R +++ b/R/utils.R @@ -5,17 +5,17 @@ readRDS_or_return_alt_data <- function(filepath, alt_return = NULL) { alt_return } -add_inv_and_port_names_if_needed <- function(data) { +add_inv_and_port_names_if_needed <- function(data, portfolio_name, investor_name) { if (!inherits(data, "data.frame")) { return(data) } if (!"portfolio_name" %in% names(data)) { - data <- dplyr::mutate(data, portfolio_name = cfg$portfolio_name, .before = dplyr::everything()) + data <- dplyr::mutate(data, portfolio_name = portfolio_name, .before = dplyr::everything()) } if (!"investor_name" %in% names(data)) { - data <- dplyr::mutate(data, investor_name = cfg$investor_name, .before = dplyr::everything()) + data <- dplyr::mutate(data, investor_name = investor_name, .before = dplyr::everything()) } data From 3f89da10879143e18ba4243f980681bb9ddaf429 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 23:08:30 +0200 Subject: [PATCH 054/132] Add argument to function --- R/run_pacta_reporting_process.R | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 0e4ffbe..3fd8ed6 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,4 +1,6 @@ -run_pacta_reporting_process <- function() { +run_pacta_reporting_process <- function( + raw_params = commandArgs(trailingOnly = TRUE) +) { suppressPackageStartupMessages({ library(pacta.portfolio.utils) library(pacta.portfolio.report) From 0ab089cbe6a62399454e211995491ac15780f7ee Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 23:10:56 +0200 Subject: [PATCH 055/132] make DESCRIPTION have a complete sentence --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index f262ad3..37fe4c4 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -15,7 +15,7 @@ Authors@R: person(given = "RMI", role = c("cph", "fnd"), email = "PACTA4investors@rmi.org")) -Description: Create Interactive Reports and Executive Summaries for PACTA +Description: Create Interactive Reports and Executive Summaries for PACTA. License: MIT + file LICENSE Encoding: UTF-8 Roxygen: list(markdown = TRUE) From a4d4fddd5513cd47397bf89ab98267d4598f427a Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 23:14:37 +0200 Subject: [PATCH 056/132] rename utility function to snake_case --- R/run_pacta_reporting_process.R | 28 ++++++++++++++-------------- R/utils.R | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 6f9f569..383df65 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -46,7 +46,7 @@ run_pacta_reporting_process <- function() { log_info("Loading PACTA results.") log_debug("Loading audit file.") - audit_file <- readRDS_or_return_alt_data( + audit_file <- read_rds_or_return_alt_data( filepath = file.path(cfg$output_dir, "audit_file.rds"), alt_return = pacta.portfolio.utils::empty_audit_file() ) @@ -57,7 +57,7 @@ run_pacta_reporting_process <- function() { ) log_debug("Loading portfolio overview.") - portfolio_overview <- readRDS_or_return_alt_data( + portfolio_overview <- read_rds_or_return_alt_data( filepath = file.path(cfg$output_dir, "overview_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_overview() ) @@ -68,7 +68,7 @@ run_pacta_reporting_process <- function() { ) log_debug("Loading emissions.") - emissions <- readRDS_or_return_alt_data( + emissions <- read_rds_or_return_alt_data( filepath = file.path(cfg$output_dir, "emissions.rds"), alt_return = pacta.portfolio.utils::empty_emissions_results() ) @@ -79,7 +79,7 @@ run_pacta_reporting_process <- function() { ) log_debug("Loading total portfolio results.") - total_portfolio <- readRDS_or_return_alt_data( + total_portfolio <- read_rds_or_return_alt_data( filepath = file.path(cfg$output_dir, "total_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) @@ -90,7 +90,7 @@ run_pacta_reporting_process <- function() { ) log_debug("Loading portfolio equity results.") - equity_results_portfolio <- readRDS_or_return_alt_data( + equity_results_portfolio <- read_rds_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) @@ -101,7 +101,7 @@ run_pacta_reporting_process <- function() { ) log_debug("Loading portfolio bonds results.") - bonds_results_portfolio <- readRDS_or_return_alt_data( + bonds_results_portfolio <- read_rds_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) @@ -112,7 +112,7 @@ run_pacta_reporting_process <- function() { ) log_debug("Loading company equity results.") - equity_results_company <- readRDS_or_return_alt_data( + equity_results_company <- read_rds_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_company.rds"), alt_return = pacta.portfolio.utils::empty_company_results() ) @@ -123,7 +123,7 @@ run_pacta_reporting_process <- function() { ) log_debug("Loading company bonds results.") - bonds_results_company <- readRDS_or_return_alt_data( + bonds_results_company <- read_rds_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_company.rds"), alt_return = pacta.portfolio.utils::empty_company_results() ) @@ -134,7 +134,7 @@ run_pacta_reporting_process <- function() { ) log_debug("Loading equity map results.") - equity_results_map <- readRDS_or_return_alt_data( + equity_results_map <- read_rds_or_return_alt_data( filepath = file.path(cfg$output_dir, "Equity_results_map.rds"), alt_return = pacta.portfolio.utils::empty_map_results() ) @@ -145,7 +145,7 @@ run_pacta_reporting_process <- function() { ) log_debug("Loading bonds map results.") - bonds_results_map <- readRDS_or_return_alt_data( + bonds_results_map <- read_rds_or_return_alt_data( filepath = file.path(cfg$output_dir, "Bonds_results_map.rds"), alt_return = pacta.portfolio.utils::empty_map_results() ) @@ -156,25 +156,25 @@ run_pacta_reporting_process <- function() { ) log_debug("Loading portfolio equity peer results.") - peers_equity_results_portfolio <- readRDS_or_return_alt_data( + peers_equity_results_portfolio <- read_rds_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading portfolio bonds peer results.") - peers_bonds_results_portfolio <- readRDS_or_return_alt_data( + peers_bonds_results_portfolio <- read_rds_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading index equity peer results.") - peers_equity_results_user <- readRDS_or_return_alt_data( + peers_equity_results_user <- read_rds_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio_ind.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading index bonds peer results.") - peers_bonds_results_user <- readRDS_or_return_alt_data( + peers_bonds_results_user <- read_rds_or_return_alt_data( filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio_ind.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) diff --git a/R/utils.R b/R/utils.R index bee56ba..9bd9426 100644 --- a/R/utils.R +++ b/R/utils.R @@ -1,4 +1,4 @@ -readRDS_or_return_alt_data <- function(filepath, alt_return = NULL) { +read_rds_or_return_alt_data <- function(filepath, alt_return = NULL) { if (file.exists(filepath)) { return(readRDS(filepath)) } From 1b748dccecc8e5d5aa900a2132d8d66c8e2b7ef0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 23:19:40 +0200 Subject: [PATCH 057/132] add `.lintr` --- .lintr | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .lintr diff --git a/.lintr b/.lintr new file mode 100644 index 0000000..7632ce7 --- /dev/null +++ b/.lintr @@ -0,0 +1,5 @@ +linters: linters_with_defaults( + object_length_linter(35), + line_length_linter(120) + ) + From 8392220f4c5fb38b9e2c91b4d7f1567bd8353308 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 23:28:09 +0200 Subject: [PATCH 058/132] Factor interactive report process to own function --- R/prepare_interactive_report.R | 94 +++++++++++++++++++++++++++++++++ R/run_pacta_reporting_process.R | 94 +-------------------------------- 2 files changed, 95 insertions(+), 93 deletions(-) create mode 100644 R/prepare_interactive_report.R diff --git a/R/prepare_interactive_report.R b/R/prepare_interactive_report.R new file mode 100644 index 0000000..12ce235 --- /dev/null +++ b/R/prepare_interactive_report.R @@ -0,0 +1,94 @@ +prepare_interactive_report <- function(){ + log_debug("Preparing to create interactive report.") + + survey_dir <- file.path(cfg$user_results_path, cfg$project_code, "survey") + real_estate_dir <- file.path(cfg$user_results_path, cfg$project_code, "real_estate") + + log_debug("Loading data frame label translations.") + dataframe_translations <- readr::read_csv( + system.file("extdata/translation/dataframe_labels.csv", package = "pacta.portfolio.report"), + col_types = readr::cols() + ) + + log_debug("Loading data frame header translations.") + header_dictionary <- readr::read_csv( + system.file("extdata/translation/dataframe_headers.csv", package = "pacta.portfolio.report"), + col_types = readr::cols() + ) + + log_debug("Loading JavaScript label translations.") + js_translations <- jsonlite::fromJSON( + txt = system.file("extdata/translation/js_labels.json", package = "pacta.portfolio.report") + ) + + log_debug("Loading sector order.") + sector_order <- readr::read_csv( + system.file("extdata/sector_order/sector_order.csv", package = "pacta.portfolio.report"), + col_types = readr::cols() + ) + + # combine config files to send to create_interactive_report() + log_trace("Defining configs and manifest.") + pacta_data_public_manifest <- + list( + creation_time_date = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$creation_time_date, + outputs_manifest = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$outputs_manifest + ) + + configs <- + list( + portfolio_config = cfg, + pacta_data_public_manifest = pacta_data_public_manifest + ) + + # workaround a bug in {config} v0.3.2 that only adds "config" class to objects it creates + class(configs$portfolio_config) <- c(class(configs$portfolio_config), "list") + + log_trace("Defining interactive report template paths.") + template_path <- system.file("templates", package = "pacta.portfolio.report") + template_dir_name <- paste(tolower(cfg$project_report_name), tolower(cfg$language_select), "template", sep = "_") + template_dir <- file.path(template_path, template_dir_name) + + log_info("Creating interactive report.") + pacta.portfolio.report::create_interactive_report( + template_dir = template_dir, + output_dir = file.path(cfg$output_dir, "report"), + survey_dir = cfg$survey_dir, + real_estate_dir = cfg$real_estate_dir, + language_select = cfg$language_select, + investor_name = cfg$investor_name, + portfolio_name = cfg$portfolio_name, + peer_group = cfg$peer_group, + start_year = cfg$start_year, + select_scenario = cfg$select_scenario, + select_scenario_other = cfg$scenario_other, + portfolio_allocation_method = cfg$portfolio_allocation_method, + scenario_geography = cfg$scenario_geography, + pacta_sectors = cfg$sector_list, + green_techs = cfg$green_techs, + tech_roadmap_sectors = cfg$tech_roadmap_sectors, + pacta_sectors_not_analysed = cfg$pacta_sectors_not_analysed, + audit_file = audit_file, + emissions = emissions, + portfolio_overview = portfolio_overview, + equity_results_portfolio = equity_results_portfolio, + bonds_results_portfolio = bonds_results_portfolio, + equity_results_company = equity_results_company, + bonds_results_company = bonds_results_company, + equity_results_map = equity_results_map, + bonds_results_map = bonds_results_map, + indices_equity_results_portfolio = indices_equity_results_portfolio, + indices_bonds_results_portfolio = indices_bonds_results_portfolio, + peers_equity_results_portfolio = peers_equity_results_portfolio, + peers_bonds_results_portfolio = peers_bonds_results_portfolio, + peers_equity_results_user = peers_equity_results_user, + peers_bonds_results_user = peers_bonds_results_user, + dataframe_translations = dataframe_translations, + js_translations = js_translations, + display_currency = cfg$display_currency, + currency_exchange_value = cfg$currency_exchange_value, + header_dictionary = header_dictionary, + sector_order = sector_order, + configs = configs + ) +} diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 383df65..308df19 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -187,99 +187,7 @@ run_pacta_reporting_process <- function() { # create interactive report ---------------------------------------------------- - log_debug("Preparing to create interactive report.") - - survey_dir <- file.path(cfg$user_results_path, cfg$project_code, "survey") - real_estate_dir <- file.path(cfg$user_results_path, cfg$project_code, "real_estate") - - log_debug("Loading data frame label translations.") - dataframe_translations <- readr::read_csv( - system.file("extdata/translation/dataframe_labels.csv", package = "pacta.portfolio.report"), - col_types = readr::cols() - ) - - log_debug("Loading data frame header translations.") - header_dictionary <- readr::read_csv( - system.file("extdata/translation/dataframe_headers.csv", package = "pacta.portfolio.report"), - col_types = readr::cols() - ) - - log_debug("Loading JavaScript label translations.") - js_translations <- jsonlite::fromJSON( - txt = system.file("extdata/translation/js_labels.json", package = "pacta.portfolio.report") - ) - - log_debug("Loading sector order.") - sector_order <- readr::read_csv( - system.file("extdata/sector_order/sector_order.csv", package = "pacta.portfolio.report"), - col_types = readr::cols() - ) - - # combine config files to send to create_interactive_report() - log_trace("Defining configs and manifest.") - pacta_data_public_manifest <- - list( - creation_time_date = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$creation_time_date, - outputs_manifest = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$outputs_manifest - ) - - configs <- - list( - portfolio_config = cfg, - pacta_data_public_manifest = pacta_data_public_manifest - ) - - # workaround a bug in {config} v0.3.2 that only adds "config" class to objects it creates - class(configs$portfolio_config) <- c(class(configs$portfolio_config), "list") - - log_trace("Defining interactive report template paths.") - template_path <- system.file("templates", package = "pacta.portfolio.report") - template_dir_name <- paste(tolower(cfg$project_report_name), tolower(cfg$language_select), "template", sep = "_") - template_dir <- file.path(template_path, template_dir_name) - - log_info("Creating interactive report.") - pacta.portfolio.report::create_interactive_report( - template_dir = template_dir, - output_dir = file.path(cfg$output_dir, "report"), - survey_dir = cfg$survey_dir, - real_estate_dir = cfg$real_estate_dir, - language_select = cfg$language_select, - investor_name = cfg$investor_name, - portfolio_name = cfg$portfolio_name, - peer_group = cfg$peer_group, - start_year = cfg$start_year, - select_scenario = cfg$select_scenario, - select_scenario_other = cfg$scenario_other, - portfolio_allocation_method = cfg$portfolio_allocation_method, - scenario_geography = cfg$scenario_geography, - pacta_sectors = cfg$sector_list, - green_techs = cfg$green_techs, - tech_roadmap_sectors = cfg$tech_roadmap_sectors, - pacta_sectors_not_analysed = cfg$pacta_sectors_not_analysed, - audit_file = audit_file, - emissions = emissions, - portfolio_overview = portfolio_overview, - equity_results_portfolio = equity_results_portfolio, - bonds_results_portfolio = bonds_results_portfolio, - equity_results_company = equity_results_company, - bonds_results_company = bonds_results_company, - equity_results_map = equity_results_map, - bonds_results_map = bonds_results_map, - indices_equity_results_portfolio = indices_equity_results_portfolio, - indices_bonds_results_portfolio = indices_bonds_results_portfolio, - peers_equity_results_portfolio = peers_equity_results_portfolio, - peers_bonds_results_portfolio = peers_bonds_results_portfolio, - peers_equity_results_user = peers_equity_results_user, - peers_bonds_results_user = peers_bonds_results_user, - dataframe_translations = dataframe_translations, - js_translations = js_translations, - display_currency = cfg$display_currency, - currency_exchange_value = cfg$currency_exchange_value, - header_dictionary = header_dictionary, - sector_order = sector_order, - configs = configs - ) - + prepare_interactive_report() # create executive summary ----------------------------------------------------- log_debug("Preparing to create executive summary.") From dc18369bb6b700a0825a0516fb340ed12a08be17 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 23:30:40 +0200 Subject: [PATCH 059/132] Factor executive summary process into own function --- R/prepare_executive_summary.R | 91 +++++++++++++++++++++++++++++++++ R/run_pacta_reporting_process.R | 90 +------------------------------- 2 files changed, 92 insertions(+), 89 deletions(-) create mode 100644 R/prepare_executive_summary.R diff --git a/R/prepare_executive_summary.R b/R/prepare_executive_summary.R new file mode 100644 index 0000000..a7d4257 --- /dev/null +++ b/R/prepare_executive_summary.R @@ -0,0 +1,91 @@ +prepare_executive_summary <- function(){ + log_debug("Preparing to create executive summary.") + + survey_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "survey")) + real_estate_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "real_estate")) + score_card_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "score_card")) + es_dir <- file.path(cfg$output_dir, "executive_summary") + if (!dir.exists(es_dir)) { + dir.create(es_dir, showWarnings = FALSE, recursive = TRUE) + } + + log_trace("Defining executive summary template paths.") + exec_summary_template_name <- paste0(cfg$project_code, "_", tolower(cfg$language_select), "_exec_summary") + exec_summary_builtin_template_path <- system.file( + "extdata", exec_summary_template_name, + package = "pacta.executive.summary" + ) + invisible(file.copy(exec_summary_builtin_template_path, cfg$output_dir, recursive = TRUE, copy.mode = FALSE)) + exec_summary_template_path <- file.path(cfg$output_dir, exec_summary_template_name) + + if ( + dir.exists(exec_summary_template_path) && ( + cfg$peer_group %in% c("assetmanager", "bank", "insurance", "pensionfund") + ) + ) { + log_debug("Preparing data for executive summary.") + data_aggregated_filtered <- + pacta.executive.summary::prep_data_executive_summary( + investor_name = cfg$investor_name, + portfolio_name = cfg$portfolio_name, + peer_group = cfg$peer_group, + start_year = cfg$start_year, + scenario_source = "GECO2021", + scenario_selected = "1.5C-Unif", + scenario_geography = "Global", + equity_market = "GlobalMarket", + portfolio_allocation_method_equity = "portfolio_weight", + portfolio_allocation_method_bonds = "portfolio_weight", + green_techs = c( + "RenewablesCap", + "HydroCap", + "NuclearCap", + "Hybrid", + "Electric", + "FuelCell", + "Hybrid_HDV", + "Electric_HDV", + "FuelCell_HDV", + "Electric Arc Furnace" + ), + equity_results_portfolio = equity_results_portfolio, + bonds_results_portfolio = bonds_results_portfolio, + peers_equity_results_aggregated = peers_equity_results_portfolio, + peers_bonds_results_aggregated = peers_bonds_results_portfolio, + peers_equity_results_individual = peers_equity_results_user, + peers_bonds_results_individual = peers_bonds_results_user, + indices_equity_results_portfolio = indices_equity_results_portfolio, + indices_bonds_results_portfolio = indices_bonds_results_portfolio, + audit_file = audit_file, + emissions_portfolio = emissions, + score_card_dir = score_card_dir + ) + + log_trace("Checking for real estate data.") + real_estate_flag <- (length(list.files(real_estate_dir)) > 0) + + log_info("Creating executive summary.") + pacta.executive.summary::render_executive_summary( + data = data_aggregated_filtered, + language = cfg$language_select, + output_dir = es_dir, + exec_summary_dir = exec_summary_template_path, + survey_dir = survey_dir, + real_estate_dir = real_estate_dir, + real_estate_flag = real_estate_flag, + score_card_dir = score_card_dir, + file_name = "template.Rmd", + investor_name = cfg$investor_name, + portfolio_name = cfg$portfolio_name, + peer_group = cfg$peer_group, + total_portfolio = total_portfolio, + scenario_selected = "1.5C-Unif", + currency_exchange_value = cfg$currency_exchange_value, + log_dir = cfg$output_dir + ) + } else { + # this is required for the online tool to know that the process has been completed. + log_debug("No executive summary created.") + invisible(file.copy(pacta.executive.summary::blank_pdf(), es_dir)) + } +} diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 308df19..0b75a06 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -190,95 +190,7 @@ run_pacta_reporting_process <- function() { prepare_interactive_report() # create executive summary ----------------------------------------------------- - log_debug("Preparing to create executive summary.") - - survey_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "survey")) - real_estate_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "real_estate")) - score_card_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "score_card")) - es_dir <- file.path(cfg$output_dir, "executive_summary") - if (!dir.exists(es_dir)) { - dir.create(es_dir, showWarnings = FALSE, recursive = TRUE) - } - - log_trace("Defining executive summary template paths.") - exec_summary_template_name <- paste0(cfg$project_code, "_", tolower(cfg$language_select), "_exec_summary") - exec_summary_builtin_template_path <- system.file( - "extdata", exec_summary_template_name, - package = "pacta.executive.summary" - ) - invisible(file.copy(exec_summary_builtin_template_path, cfg$output_dir, recursive = TRUE, copy.mode = FALSE)) - exec_summary_template_path <- file.path(cfg$output_dir, exec_summary_template_name) - - if ( - dir.exists(exec_summary_template_path) && ( - cfg$peer_group %in% c("assetmanager", "bank", "insurance", "pensionfund") - ) - ) { - log_debug("Preparing data for executive summary.") - data_aggregated_filtered <- - pacta.executive.summary::prep_data_executive_summary( - investor_name = cfg$investor_name, - portfolio_name = cfg$portfolio_name, - peer_group = cfg$peer_group, - start_year = cfg$start_year, - scenario_source = "GECO2021", - scenario_selected = "1.5C-Unif", - scenario_geography = "Global", - equity_market = "GlobalMarket", - portfolio_allocation_method_equity = "portfolio_weight", - portfolio_allocation_method_bonds = "portfolio_weight", - green_techs = c( - "RenewablesCap", - "HydroCap", - "NuclearCap", - "Hybrid", - "Electric", - "FuelCell", - "Hybrid_HDV", - "Electric_HDV", - "FuelCell_HDV", - "Electric Arc Furnace" - ), - equity_results_portfolio = equity_results_portfolio, - bonds_results_portfolio = bonds_results_portfolio, - peers_equity_results_aggregated = peers_equity_results_portfolio, - peers_bonds_results_aggregated = peers_bonds_results_portfolio, - peers_equity_results_individual = peers_equity_results_user, - peers_bonds_results_individual = peers_bonds_results_user, - indices_equity_results_portfolio = indices_equity_results_portfolio, - indices_bonds_results_portfolio = indices_bonds_results_portfolio, - audit_file = audit_file, - emissions_portfolio = emissions, - score_card_dir = score_card_dir - ) - - log_trace("Checking for real estate data.") - real_estate_flag <- (length(list.files(real_estate_dir)) > 0) - - log_info("Creating executive summary.") - pacta.executive.summary::render_executive_summary( - data = data_aggregated_filtered, - language = cfg$language_select, - output_dir = es_dir, - exec_summary_dir = exec_summary_template_path, - survey_dir = survey_dir, - real_estate_dir = real_estate_dir, - real_estate_flag = real_estate_flag, - score_card_dir = score_card_dir, - file_name = "template.Rmd", - investor_name = cfg$investor_name, - portfolio_name = cfg$portfolio_name, - peer_group = cfg$peer_group, - total_portfolio = total_portfolio, - scenario_selected = "1.5C-Unif", - currency_exchange_value = cfg$currency_exchange_value, - log_dir = cfg$output_dir - ) - } else { - # this is required for the online tool to know that the process has been completed. - log_debug("No executive summary created.") - invisible(file.copy(pacta.executive.summary::blank_pdf(), es_dir)) - } + prepare_executive_summary() log_info("Portfolio report finished.") } From a51f9c905fdb573ae69db7ac7b6cf843b1ccf6b0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 23:33:09 +0200 Subject: [PATCH 060/132] Increment version number to 0.0.0.9001 --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 37fe4c4..eaf4404 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,6 +1,6 @@ Package: workflow.pacta.report Title: Reporting functionality for PACTA -Version: 0.0.0.9000 +Version: 0.0.0.9001 Authors@R: c(person(given = "Alex", family = "Axthelm", From f3b42b311cd104feee433dcb443c0c572add623b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Wed, 29 May 2024 23:34:12 +0200 Subject: [PATCH 061/132] Render Docs --- man/workflow.pacta.report-package.Rd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/man/workflow.pacta.report-package.Rd b/man/workflow.pacta.report-package.Rd index 909c19e..c57a881 100644 --- a/man/workflow.pacta.report-package.Rd +++ b/man/workflow.pacta.report-package.Rd @@ -6,7 +6,7 @@ \alias{workflow.pacta.report-package} \title{workflow.pacta.report: Reporting functionality for PACTA} \description{ -Create Interactive Reports and Executive Summaries for PACTA +Create Interactive Reports and Executive Summaries for PACTA. } \author{ \strong{Maintainer}: Alex Axthelm \email{aaxthelm@rmi.org} (\href{https://orcid.org/0000-0001-8579-8565}{ORCID}) [contractor] From 6a4515913d6ba76824fc5b9e59a73c9368d8aac3 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 09:14:49 +0200 Subject: [PATCH 062/132] Download prepared index benchmarks in tests --- .github/workflows/test.yml | 19 ++++++++++++++++++- tests/config/default_2022Q4.json | 1 + tests/config/default_2023Q4.json | 1 + tests/config/full_params_2023Q4.json | 1 + 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 07cb33f..2c7121d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,6 +43,10 @@ jobs: echo "pacta-data-url=$PACTA_DATA_URL" echo "pacta-data-url=$PACTA_DATA_URL" >> "$GITHUB_OUTPUT" + BENCHMARKS_URL="$(jq -rc '.benchmarksURL' $config_file)" + echo "benchmars-url=$BENCHMARKS_URL" + echo "benchmars-url=$BENCHMARKS_URL" >> "$GITHUB_OUTPUT" + # includes handling for null/missing keys PARAMETERS="$(jq -rc '.parameters | select( . != null )' $config_file)" echo "parameters=$PARAMETERS" @@ -64,13 +68,26 @@ jobs: with: # azcliversion: 2.30.0 inlineScript: | - echo "pacta_data_dir: $PACTA_DATA_DATE_DIR" az storage copy \ --source "$PACTA_DATA_URL/*" \ --destination "pacta-data" \ --recursive \ --exclude-pattern "*.sqlite" + # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version + - name: Download Benchmark Results + uses: azure/CLI@v2 + env: + BENCHMARKS_URL: ${{ steps.prepare.outputs.benchmarks-url }} + with: + # azcliversion: 2.30.0 + inlineScript: | + az storage copy \ + --source "$BENCHMARKS_URL/*" \ + --destination "pacta-data" \ + --recursive \ + --exclude-pattern "*.sqlite" + - name: Run PACTA Analysis env: FULL_IMAGE_NAME: ${{ inputs.full-image-name }} diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json index dfac979..a7560b8 100644 --- a/tests/config/default_2022Q4.json +++ b/tests/config/default_2022Q4.json @@ -1,6 +1,7 @@ { "holdingsDate": "2022Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", + "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", "parameters": { "portfolio_files": "default_portfolio.csv", "inherit": "GENERAL_2022Q4" diff --git a/tests/config/default_2023Q4.json b/tests/config/default_2023Q4.json index 83e2c06..0df6931 100644 --- a/tests/config/default_2023Q4.json +++ b/tests/config/default_2023Q4.json @@ -1,6 +1,7 @@ { "holdingsDate": "2023Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", + "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", "parameters": { "portfolio_files": "default_portfolio.csv", "inherit": "GENERAL_2023Q4" diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json index 9786e89..3ab5998 100644 --- a/tests/config/full_params_2023Q4.json +++ b/tests/config/full_params_2023Q4.json @@ -1,6 +1,7 @@ { "holdingsDate": "2023Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", + "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", "parameters": { "equity_market_list": ["GlobalMarket", "DevelopedMarket", "EmergingMarket"], "scenario_geographies_list": ["Global", "GlobalAggregate", "NonOECD", "OECD"], From f7085622a90d2232c1cba4f6325a85d3679e4843 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 09:57:11 +0200 Subject: [PATCH 063/132] Fix typo in variable name --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2c7121d..d72e49e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,8 +44,8 @@ jobs: echo "pacta-data-url=$PACTA_DATA_URL" >> "$GITHUB_OUTPUT" BENCHMARKS_URL="$(jq -rc '.benchmarksURL' $config_file)" - echo "benchmars-url=$BENCHMARKS_URL" - echo "benchmars-url=$BENCHMARKS_URL" >> "$GITHUB_OUTPUT" + echo "benchmarks-url=$BENCHMARKS_URL" + echo "benchmarks-url=$BENCHMARKS_URL" >> "$GITHUB_OUTPUT" # includes handling for null/missing keys PARAMETERS="$(jq -rc '.parameters | select( . != null )' $config_file)" From 52200088347ee2dad745d6d7e965e716505d4ff7 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 10:57:25 +0200 Subject: [PATCH 064/132] change default logging level --- R/run_pacta_reporting_process.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 77fb139..ee2ef60 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -3,7 +3,7 @@ run_pacta_reporting_process <- function( ) { # defaulting to WARN to maintain current (silent) behavior. - logger::log_threshold(Sys.getenv("LOG_LEVEL", "WARN")) + logger::log_threshold(Sys.getenv("LOG_LEVEL", "INFO")) logger::log_formatter(logger::formatter_glue) # ------------------------------------------------------------------------- From 2f229f01fcc55a502ca9895cf70773ff82a6ab73 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 11:16:34 +0200 Subject: [PATCH 065/132] Use dev version of dependency --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 37fe4c4..46e9df1 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -31,5 +31,5 @@ Imports: readr Remotes: RMI-PACTA/pacta.executive.summary, - RMI-PACTA/pacta.portfolio.report, + RMI-PACTA/pacta.portfolio.report@issues-found-in-workflow.pacta.report, RMI-PACTA/pacta.portfolio.utils From 6453df7c160183be5987d9daf3ac159ee4e82963 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 10:54:45 +0200 Subject: [PATCH 066/132] Pass `cfg` to prep functions --- R/prepare_executive_summary.R | 4 +++- R/prepare_interactive_report.R | 4 +++- R/run_pacta_reporting_process.R | 8 ++++++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/R/prepare_executive_summary.R b/R/prepare_executive_summary.R index a7d4257..7366dc0 100644 --- a/R/prepare_executive_summary.R +++ b/R/prepare_executive_summary.R @@ -1,4 +1,6 @@ -prepare_executive_summary <- function(){ +prepare_executive_summary <- function( + cfg +) { log_debug("Preparing to create executive summary.") survey_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "survey")) diff --git a/R/prepare_interactive_report.R b/R/prepare_interactive_report.R index 12ce235..84eb980 100644 --- a/R/prepare_interactive_report.R +++ b/R/prepare_interactive_report.R @@ -1,4 +1,6 @@ -prepare_interactive_report <- function(){ +prepare_interactive_report <- function( + cfg +){ log_debug("Preparing to create interactive report.") survey_dir <- file.path(cfg$user_results_path, cfg$project_code, "survey") diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 59dd2be..57c5a26 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -189,10 +189,14 @@ run_pacta_reporting_process <- function( # create interactive report ---------------------------------------------------- - prepare_interactive_report() + prepare_interactive_report( + cfg = cfg + ) # create executive summary ----------------------------------------------------- - prepare_executive_summary() + prepare_executive_summary( + cfg = cfg + ) log_info("Portfolio report finished.") } From 8064bfbf4c58e7e14183d9dbb347123b54964547 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 12:01:35 +0200 Subject: [PATCH 067/132] Add benchmark arguments to subfunctions --- R/run_pacta_reporting_process.R | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 57c5a26..bb3160a 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -190,12 +190,24 @@ run_pacta_reporting_process <- function( # create interactive report ---------------------------------------------------- prepare_interactive_report( - cfg = cfg + cfg = cfg, + indices_bonds_results_portfolio = indices_bonds_results_portfolio, + indices_equity_results_portfolio = indices_equity_results_portfolio, + peers_bonds_results_portfolio = peers_bonds_results_portfolio, + peers_bonds_results_user = peers_bonds_results_user, + peers_equity_results_portfolio = peers_equity_results_portfolio, + peers_equity_results_user = peers_equity_results_user ) # create executive summary ----------------------------------------------------- prepare_executive_summary( - cfg = cfg + cfg = cfg, + indices_bonds_results_portfolio = indices_bonds_results_portfolio, + indices_equity_results_portfolio = indices_equity_results_portfolio, + peers_bonds_results_portfolio = peers_bonds_results_portfolio, + peers_bonds_results_user = peers_bonds_results_user, + peers_equity_results_portfolio = peers_equity_results_portfolio, + peers_equity_results_user = peers_equity_results_user ) log_info("Portfolio report finished.") From e762f2eabd3f93123ce2dcbe1eeefe5ab43cb55d Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 12:09:20 +0200 Subject: [PATCH 068/132] Add arguments for missing objects in report --- R/prepare_interactive_report.R | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/R/prepare_interactive_report.R b/R/prepare_interactive_report.R index 84eb980..8b8fbfc 100644 --- a/R/prepare_interactive_report.R +++ b/R/prepare_interactive_report.R @@ -1,5 +1,20 @@ prepare_interactive_report <- function( - cfg + cfg, + audit_file, + emissions, + portfolio_overview, + equity_results_portfolio, + bonds_results_portfolio, + equity_results_company, + bonds_results_company, + equity_results_map, + bonds_results_map, + indices_bonds_results_portfolio, + indices_equity_results_portfolio, + peers_bonds_results_portfolio, + peers_bonds_results_user, + peers_equity_results_portfolio, + peers_equity_results_user ){ log_debug("Preparing to create interactive report.") From 1b5c2238edd214f1e7cceb31370b32ebe594160b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 12:10:50 +0200 Subject: [PATCH 069/132] Add arguments for missing objects in summary --- R/prepare_executive_summary.R | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/R/prepare_executive_summary.R b/R/prepare_executive_summary.R index 7366dc0..da4a8a5 100644 --- a/R/prepare_executive_summary.R +++ b/R/prepare_executive_summary.R @@ -1,5 +1,16 @@ prepare_executive_summary <- function( - cfg + cfg, + total_portfolio, + equity_results_portfolio, + bonds_results_portfolio, + indices_equity_results_portfolio, + indices_bonds_results_portfolio, + audit_file, + emissions, + peers_bonds_results_portfolio, + peers_bonds_results_user, + peers_equity_results_portfolio, + peers_equity_results_user ) { log_debug("Preparing to create executive summary.") From 4f8a071d172aa6abee1ca84c3138df7c540e9c77 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 12:12:03 +0200 Subject: [PATCH 070/132] Update function calls --- R/run_pacta_reporting_process.R | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index bb3160a..fe16306 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -191,6 +191,15 @@ run_pacta_reporting_process <- function( prepare_interactive_report( cfg = cfg, + audit_file = audit_file, + emissions = emissions, + portfolio_overview = portfolio_overview, + equity_results_portfolio = equity_results_portfolio, + bonds_results_portfolio = bonds_results_portfolio, + equity_results_company = equity_results_company, + bonds_results_company = bonds_results_company, + equity_results_map = equity_results_map, + bonds_results_map = bonds_results_map, indices_bonds_results_portfolio = indices_bonds_results_portfolio, indices_equity_results_portfolio = indices_equity_results_portfolio, peers_bonds_results_portfolio = peers_bonds_results_portfolio, @@ -202,8 +211,13 @@ run_pacta_reporting_process <- function( # create executive summary ----------------------------------------------------- prepare_executive_summary( cfg = cfg, - indices_bonds_results_portfolio = indices_bonds_results_portfolio, + total_portfolio = total_portfolio, + equity_results_portfolio = equity_results_portfolio, + bonds_results_portfolio = bonds_results_portfolio, indices_equity_results_portfolio = indices_equity_results_portfolio, + indices_bonds_results_portfolio = indices_bonds_results_portfolio, + audit_file = audit_file, + emissions = emissions, peers_bonds_results_portfolio = peers_bonds_results_portfolio, peers_bonds_results_user = peers_bonds_results_user, peers_equity_results_portfolio = peers_equity_results_portfolio, From 000ad1398dd6f057a58fce42f29cee0dbdfe90f8 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 12:18:57 +0200 Subject: [PATCH 071/132] Linting (indentation) --- R/prepare_executive_summary.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/prepare_executive_summary.R b/R/prepare_executive_summary.R index da4a8a5..5ce2aa9 100644 --- a/R/prepare_executive_summary.R +++ b/R/prepare_executive_summary.R @@ -60,7 +60,7 @@ prepare_executive_summary <- function( "Electric_HDV", "FuelCell_HDV", "Electric Arc Furnace" - ), + ), equity_results_portfolio = equity_results_portfolio, bonds_results_portfolio = bonds_results_portfolio, peers_equity_results_aggregated = peers_equity_results_portfolio, From a62049252c50b692a1f3c9009a40875ac987822b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 12:19:41 +0200 Subject: [PATCH 072/132] linting (brace) --- R/prepare_interactive_report.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/prepare_interactive_report.R b/R/prepare_interactive_report.R index 8b8fbfc..a984840 100644 --- a/R/prepare_interactive_report.R +++ b/R/prepare_interactive_report.R @@ -15,7 +15,7 @@ prepare_interactive_report <- function( peers_bonds_results_user, peers_equity_results_portfolio, peers_equity_results_user -){ +) { log_debug("Preparing to create interactive report.") survey_dir <- file.path(cfg$user_results_path, cfg$project_code, "survey") From abb289be1b406b6e32a3a7deb6ed6bb564e89323 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 12:35:17 +0200 Subject: [PATCH 073/132] remove unused variables --- R/prepare_interactive_report.R | 3 --- 1 file changed, 3 deletions(-) diff --git a/R/prepare_interactive_report.R b/R/prepare_interactive_report.R index a984840..b87228e 100644 --- a/R/prepare_interactive_report.R +++ b/R/prepare_interactive_report.R @@ -18,9 +18,6 @@ prepare_interactive_report <- function( ) { log_debug("Preparing to create interactive report.") - survey_dir <- file.path(cfg$user_results_path, cfg$project_code, "survey") - real_estate_dir <- file.path(cfg$user_results_path, cfg$project_code, "real_estate") - log_debug("Loading data frame label translations.") dataframe_translations <- readr::read_csv( system.file("extdata/translation/dataframe_labels.csv", package = "pacta.portfolio.report"), From ab560208d4b046844023f062213175db79831d39 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 12:55:53 +0200 Subject: [PATCH 074/132] Download results from workflow.pacta, not create --- .github/workflows/test.yml | 54 ++++++++++------------------ tests/config/default_2022Q4.json | 1 + tests/config/default_2023Q4.json | 1 + tests/config/full_params_2023Q4.json | 1 + 4 files changed, 21 insertions(+), 36 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d72e49e..c3647ee 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,10 @@ jobs: echo "benchmarks-url=$BENCHMARKS_URL" echo "benchmarks-url=$BENCHMARKS_URL" >> "$GITHUB_OUTPUT" + RESULTS_URL="$(jq -rc '.resultsURL' $config_file)" + echo "results-url=$RESULTS_URL" + echo "results-url=$RESULTS_URL" >> "$GITHUB_OUTPUT" + # includes handling for null/missing keys PARAMETERS="$(jq -rc '.parameters | select( . != null )' $config_file)" echo "parameters=$PARAMETERS" @@ -74,6 +78,20 @@ jobs: --recursive \ --exclude-pattern "*.sqlite" + # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version + - name: Download outputs from worfklow.pacta + uses: azure/CLI@v2 + env: + RESULTS_URL: ${{ steps.prepare.outputs.results-url }} + with: + # azcliversion: 2.30.0 + inlineScript: | + az storage copy \ + --source "$BENCHMARKS_URL/*" \ + --destination "outputs/" \ + --recursive \ + --exclude-pattern "*.sqlite" + # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version - name: Download Benchmark Results uses: azure/CLI@v2 @@ -88,42 +106,6 @@ jobs: --recursive \ --exclude-pattern "*.sqlite" - - name: Run PACTA Analysis - env: - FULL_IMAGE_NAME: ${{ inputs.full-image-name }} - OUTPUT_DIR: output_dir - PACTA_DATA_DIR: pacta-data - WORKSPACE: ${{ github.workspace }} - PARAMETERS: ${{ steps.prepare.outputs.parameters }} - run: | - mkdir -p "${WORKSPACE}/${OUTPUT_DIR}" - chmod -R 777 "${WORKSPACE}/${OUTPUT_DIR}" - docker run \ - --network none \ - --env LOG_LEVEL=DEBUG \ - --env PORTFOLIO_DIR=/workflow.pacta/portfolios/ \ - --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ - --mount type=bind,source=${WORKSPACE}/${OUTPUT_DIR},target=/mnt/output_dir \ - ghcr.io/rmi-pacta/workflow.pacta:main \ - $PARAMETERS - - - name: Run Docker Image - env: - FULL_IMAGE_NAME: ${{ inputs.full-image-name }} - OUTPUT_DIR: output_dir - PACTA_DATA_DIR: pacta-data - WORKSPACE: ${{ github.workspace }} - PARAMETERS: ${{ steps.prepare.outputs.parameters }} - run: | - docker run \ - --network none \ - --env LOG_LEVEL=DEBUG \ - --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ - --mount type=bind,source=${WORKSPACE}/${OUTPUT_DIR},target=/mnt/output_dir \ - $FULL_IMAGE_NAME - # $FULL_IMAGE_NAME \ - # $PARAMETERS - - name: List outputs run: | ls -lR output_dir diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json index a7560b8..5d0eaed 100644 --- a/tests/config/default_2022Q4.json +++ b/tests/config/default_2022Q4.json @@ -1,6 +1,7 @@ { "holdingsDate": "2022Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-report-results/17/merge/38/2/default_2022Q4/workflow.pacta.report/output_dir/report/report/index.html", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", "parameters": { "portfolio_files": "default_portfolio.csv", diff --git a/tests/config/default_2023Q4.json b/tests/config/default_2023Q4.json index 0df6931..0374f14 100644 --- a/tests/config/default_2023Q4.json +++ b/tests/config/default_2023Q4.json @@ -1,6 +1,7 @@ { "holdingsDate": "2023Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-report-results/17/merge/38/2/default_2023Q4/workflow.pacta.report/output_dir/report/report/index.html", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", "parameters": { "portfolio_files": "default_portfolio.csv", diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json index 3ab5998..b50da78 100644 --- a/tests/config/full_params_2023Q4.json +++ b/tests/config/full_params_2023Q4.json @@ -1,6 +1,7 @@ { "holdingsDate": "2023Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-report-results/17/merge/38/2/default_2023Q4/workflow.pacta.report/output_dir/report/report/index.html", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", "parameters": { "equity_market_list": ["GlobalMarket", "DevelopedMarket", "EmergingMarket"], From 301ff1b23f36f85b4a5845afd88ef884d955818b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 12:58:16 +0200 Subject: [PATCH 075/132] add back in testing step --- .github/workflows/test.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c3647ee..63533e8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -106,6 +106,23 @@ jobs: --recursive \ --exclude-pattern "*.sqlite" + - name: Run Docker Image + env: + FULL_IMAGE_NAME: ${{ inputs.full-image-name }} + OUTPUT_DIR: output_dir + PACTA_DATA_DIR: pacta-data + WORKSPACE: ${{ github.workspace }} + PARAMETERS: ${{ steps.prepare.outputs.parameters }} + run: | + docker run \ + --network none \ + --env LOG_LEVEL=DEBUG \ + --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ + --mount type=bind,source=${WORKSPACE}/${OUTPUT_DIR},target=/mnt/output_dir \ + $FULL_IMAGE_NAME + # $FULL_IMAGE_NAME \ + # $PARAMETERS + - name: List outputs run: | ls -lR output_dir From 4859fe56faeecfc71202d74e9573a7d0a8fe555a Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 13:09:29 +0200 Subject: [PATCH 076/132] Fir envvar reference --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 63533e8..0687658 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,7 +79,7 @@ jobs: --exclude-pattern "*.sqlite" # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version - - name: Download outputs from worfklow.pacta + - name: Download outputs from workflow.pacta uses: azure/CLI@v2 env: RESULTS_URL: ${{ steps.prepare.outputs.results-url }} @@ -87,7 +87,7 @@ jobs: # azcliversion: 2.30.0 inlineScript: | az storage copy \ - --source "$BENCHMARKS_URL/*" \ + --source "$RESULTS_URL/*" \ --destination "outputs/" \ --recursive \ --exclude-pattern "*.sqlite" From 788ab4e8013895600769c0b255d135a42de071d1 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 14:12:59 +0200 Subject: [PATCH 077/132] Fix path --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0687658..ae4733c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -88,7 +88,7 @@ jobs: inlineScript: | az storage copy \ --source "$RESULTS_URL/*" \ - --destination "outputs/" \ + --destination "output_dir/" \ --recursive \ --exclude-pattern "*.sqlite" From cdc9b94a4f3e246c7153444a61454a0773f1df1a Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 14:26:51 +0200 Subject: [PATCH 078/132] Don't download full pacta-data --- .github/workflows/test.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ae4733c..0bde87a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,10 +39,6 @@ jobs: echo "holdings-date=$HOLDINGS_DATE" echo "holdings-date=$HOLDINGS_DATE" >> "$GITHUB_OUTPUT" - PACTA_DATA_URL="$(jq -rc '.pactaDataURL' $config_file)" - echo "pacta-data-url=$PACTA_DATA_URL" - echo "pacta-data-url=$PACTA_DATA_URL" >> "$GITHUB_OUTPUT" - BENCHMARKS_URL="$(jq -rc '.benchmarksURL' $config_file)" echo "benchmarks-url=$BENCHMARKS_URL" echo "benchmarks-url=$BENCHMARKS_URL" >> "$GITHUB_OUTPUT" @@ -64,20 +60,6 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version - - name: Download pacta-data - uses: azure/CLI@v2 - env: - PACTA_DATA_URL: ${{ steps.prepare.outputs.pacta-data-url }} - with: - # azcliversion: 2.30.0 - inlineScript: | - az storage copy \ - --source "$PACTA_DATA_URL/*" \ - --destination "pacta-data" \ - --recursive \ - --exclude-pattern "*.sqlite" - # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version - name: Download outputs from workflow.pacta uses: azure/CLI@v2 From 0964cb7b8b39605f9145bc708433ab1e79e5ba73 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 14:55:49 +0200 Subject: [PATCH 079/132] Fix paths to testing results from `workflow.pacta` --- tests/config/default_2022Q4.json | 2 +- tests/config/default_2023Q4.json | 2 +- tests/config/full_params_2023Q4.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/config/default_2022Q4.json b/tests/config/default_2022Q4.json index 5d0eaed..d52e8ec 100644 --- a/tests/config/default_2022Q4.json +++ b/tests/config/default_2022Q4.json @@ -1,7 +1,7 @@ { "holdingsDate": "2022Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2022Q4_20240426T113151Z", - "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-report-results/17/merge/38/2/default_2022Q4/workflow.pacta.report/output_dir/report/report/index.html", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2022Q4/output_dir", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2022Q4_20240529T002407Z", "parameters": { "portfolio_files": "default_portfolio.csv", diff --git a/tests/config/default_2023Q4.json b/tests/config/default_2023Q4.json index 0374f14..2a6ec42 100644 --- a/tests/config/default_2023Q4.json +++ b/tests/config/default_2023Q4.json @@ -1,7 +1,7 @@ { "holdingsDate": "2023Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", - "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-report-results/17/merge/38/2/default_2023Q4/workflow.pacta.report/output_dir/report/report/index.html", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/default_2023Q4/output_dir", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", "parameters": { "portfolio_files": "default_portfolio.csv", diff --git a/tests/config/full_params_2023Q4.json b/tests/config/full_params_2023Q4.json index b50da78..3463007 100644 --- a/tests/config/full_params_2023Q4.json +++ b/tests/config/full_params_2023Q4.json @@ -1,7 +1,7 @@ { "holdingsDate": "2023Q4", "pactaDataURL": "https://pactadatadev.file.core.windows.net/workflow-data-preparation-outputs/2023Q4_20240424T120055Z", - "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-report-results/17/merge/38/2/default_2023Q4/workflow.pacta.report/output_dir/report/report/index.html", + "resultsURL": "https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/73/merge/latest/full_params_2023Q4/output_dir", "benchmarksURL": "https://pactadatadev.file.core.windows.net/workflow-prepare-pacta-indices-outputs/2023Q4_20240529T002355Z", "parameters": { "equity_market_list": ["GlobalMarket", "DevelopedMarket", "EmergingMarket"], From fbf4c3409bd2527bb183340ca21dd0f20aa32b95 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 15:02:05 +0200 Subject: [PATCH 080/132] Download manifest of `pacta-data` --- .github/workflows/test.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0bde87a..9b3f42e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,6 +60,20 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version + - name: Download pacta-data + uses: azure/CLI@v2 + env: + PACTA_DATA_URL: ${{ steps.prepare.outputs.pacta-data-url }} + with: + # azcliversion: 2.30.0 + inlineScript: | + az storage copy \ + --source "$PACTA_DATA_URL/manifest.json" \ + --destination "pacta-data/manifest.json" \ + --recursive \ + --exclude-pattern "*.sqlite" + # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version - name: Download outputs from workflow.pacta uses: azure/CLI@v2 From a484554657caeb1c7cc258e0f45f9b75e39d401e Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 15:06:20 +0200 Subject: [PATCH 081/132] Read pacta data path from test config --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9b3f42e..bf898f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,10 @@ jobs: echo "holdings-date=$HOLDINGS_DATE" echo "holdings-date=$HOLDINGS_DATE" >> "$GITHUB_OUTPUT" + PACTA_DATA_URL="$(jq -rc '.pactaDataURL' $config_file)" + echo "pacta-data-url=$PACTA_DATA_URL" + echo "pacta-data-url=$PACTA_DATA_URL" >> "$GITHUB_OUTPUT" + BENCHMARKS_URL="$(jq -rc '.benchmarksURL' $config_file)" echo "benchmarks-url=$BENCHMARKS_URL" echo "benchmarks-url=$BENCHMARKS_URL" >> "$GITHUB_OUTPUT" From 0b81bde0498648d5f2e1a72e7f5ab539674e7d68 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 15:26:33 +0200 Subject: [PATCH 082/132] Export reports to own directory In response to permissions error in current arrangement --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf898f8..34bc420 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -110,15 +110,19 @@ jobs: env: FULL_IMAGE_NAME: ${{ inputs.full-image-name }} OUTPUT_DIR: output_dir + REPORT_DIR: report PACTA_DATA_DIR: pacta-data WORKSPACE: ${{ github.workspace }} PARAMETERS: ${{ steps.prepare.outputs.parameters }} run: | + mkdir -p "$REPORT_DIR" + chmod -R 777 "$REPORT_DIR" docker run \ --network none \ --env LOG_LEVEL=DEBUG \ --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ --mount type=bind,source=${WORKSPACE}/${OUTPUT_DIR},target=/mnt/output_dir \ + --mount type=bind,source=${WORKSPACE}/${REPORT_DIR},target=/mnt/report_dir \ $FULL_IMAGE_NAME # $FULL_IMAGE_NAME \ # $PARAMETERS From 6f334b93f76348c30ccd3183d10b8d041de0de1c Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 15:37:58 +0200 Subject: [PATCH 083/132] Update calling function --- R/prepare_interactive_report.R | 2 +- input_dir/default_config.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/R/prepare_interactive_report.R b/R/prepare_interactive_report.R index b87228e..ae9f0ab 100644 --- a/R/prepare_interactive_report.R +++ b/R/prepare_interactive_report.R @@ -66,7 +66,7 @@ prepare_interactive_report <- function( log_info("Creating interactive report.") pacta.portfolio.report::create_interactive_report( template_dir = template_dir, - output_dir = file.path(cfg$output_dir, "report"), + output_dir = cfg$report_dir, survey_dir = cfg$survey_dir, real_estate_dir = cfg$real_estate_dir, language_select = cfg$language_select, diff --git a/input_dir/default_config.json b/input_dir/default_config.json index 642675e..bc412ec 100644 --- a/input_dir/default_config.json +++ b/input_dir/default_config.json @@ -2,6 +2,7 @@ "data_dir": "/mnt/pacta-data/", "portfolio_path": "/mnt/input_dir/default_portfolio.csv", "output_dir": "/mnt/output_dir", + "report_dir": "/mnt/report_dir", "start_year": 2023, "equity_market_list": [ "GlobalMarket", From 219822a9a5c907db1d4b70ee9ce65084841b300f Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 15:57:30 +0200 Subject: [PATCH 084/132] Upload Reports --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34bc420..93d851b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -139,12 +139,12 @@ jobs: GITHUB_REF_NAME: ${{ github.ref_name}} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} GITHUB_RUN_NUMBER: ${{ github.run_number }} - OUTPUTS_DIR: "output_dir" + REPORT_DIR: report RESULTS_URL: ${{ inputs.results-url }} with: inlineScript: | unique_directory="$RESULTS_URL/$GITHUB_REF_NAME/$GITHUB_RUN_NUMBER/$GITHUB_RUN_ATTEMPT/$CONFIG_NAME" az storage copy \ - --source "$outputs_dir" \ + --source "$REPORT_DIR" \ --destination "$unique_directory" \ --recursive From 8528d3b64badf3a5c4be4bb25baff39f5283fa90 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 17:26:20 +0200 Subject: [PATCH 085/132] Add comment with links --- .github/workflows/docker.yml | 8 +++++++ .github/workflows/test.yml | 46 ++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1b9a4d6..3011bd0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,3 +29,11 @@ jobs: with: full-image-name: ${{ needs.docker.outputs.full-image-name }} config-name: ${{ matrix.config-name }} + + add-comment: + if: ${{ github.event_name == 'pull_request' }} + uses: RMI-PACTA/actions/.github/workflows/add-comment-table.yml@main + needs: [test] + with: + header: "Report Links" + artifact-prefix: report-comment-json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 93d851b..5beacf0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -133,6 +133,7 @@ jobs: # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version - name: Upload results to blob store + id: upload uses: azure/CLI@v2 env: CONFIG_NAME: ${{ inputs.config-name }} @@ -148,3 +149,48 @@ jobs: --source "$REPORT_DIR" \ --destination "$unique_directory" \ --recursive + echo "report-url=${unique_directory}/index.html" >> "$GITHUB_OUTPUT" + + - name: Prepare comment artifact + id: prepare-artifact + env: + commit_time: ${{ github.event.pull_request.updated_at }} + config_name: ${{ inputs.config-name }} + full_image_name: ${{ inputs.full-image-name }} + git_sha: ${{ github.event.pull_request.head.sha }} + report_url: ${{ steps.upload.outputs.report-url }} + summary_url: ${{ env.summary-url }} + run: | + mkdir -p /tmp/report-comment-json + json_filename=$( \ + echo "report-comment-json-merge-$full_image_name-$config_name.json" | \ + tr '/' '-' \ + ) + echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" + json_file="/tmp/report-comment-json/$json_filename" + jq \ + -n \ + --arg commit_time "$commit_time" \ + --arg config_name "$config_name" \ + --arg git_sha "$git_sha" \ + --arg report_url "$report_url" \ + --arg summary_url "$summary_url" \ + --arg full_image_name "$full_image_name" \ + '{ + "commit_time": $commit_time, + "git_sha": $git_sha, + "config": $config_name, + "report": ("[Report](" + $report_url + ")"), + "summary": (if (("" + $summary_url) == "") then null else "[Summary](" + $summary_url + ")" end), + "image": ("`" + $full_image_name + "`") + }' \ + >> $json_file + cat "$json_file" + + - name: Upload comment JSON + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.prepare-artifact.outputs.json-filename }} + path: /tmp/report-comment-json/* + if-no-files-found: error + retention-days: 1 From 3ee176cf9179b6aa7c270b9905b4bdd1bd2fe0e2 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 17:35:10 +0200 Subject: [PATCH 086/132] Remove special characters from artifact name --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5beacf0..0c01f48 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -164,7 +164,7 @@ jobs: mkdir -p /tmp/report-comment-json json_filename=$( \ echo "report-comment-json-merge-$full_image_name-$config_name.json" | \ - tr '/' '-' \ + tr '/:._' '-' \ ) echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" json_file="/tmp/report-comment-json/$json_filename" From 1c051777044d4da62f6a59ed5a0ae5a86104caa6 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 17:46:57 +0200 Subject: [PATCH 087/132] Update comment json path --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0c01f48..e8813ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -161,13 +161,13 @@ jobs: report_url: ${{ steps.upload.outputs.report-url }} summary_url: ${{ env.summary-url }} run: | - mkdir -p /tmp/report-comment-json + mkdir -p /tmp/comment-json json_filename=$( \ echo "report-comment-json-merge-$full_image_name-$config_name.json" | \ tr '/:._' '-' \ ) echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" - json_file="/tmp/report-comment-json/$json_filename" + json_file="/tmp/comment-json/$json_filename" jq \ -n \ --arg commit_time "$commit_time" \ @@ -191,6 +191,6 @@ jobs: uses: actions/upload-artifact@v4 with: name: ${{ steps.prepare-artifact.outputs.json-filename }} - path: /tmp/report-comment-json/* + path: /tmp/comment-json/* if-no-files-found: error retention-days: 1 From bc263c9b15e7c9a6bf8ea4be1483fb19593f2a61 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 17:52:56 +0200 Subject: [PATCH 088/132] fix filenames --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e8813ed..0093e00 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -163,10 +163,10 @@ jobs: run: | mkdir -p /tmp/comment-json json_filename=$( \ - echo "report-comment-json-merge-$full_image_name-$config_name.json" | \ + echo "report-comment-json-merge-$full_image_name-$config_name" | \ tr '/:._' '-' \ ) - echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" + echo "json-filename=${json_filename}.json" >> "$GITHUB_OUTPUT" json_file="/tmp/comment-json/$json_filename" jq \ -n \ From f8deff4da57265aa5cab6e4e61a808f62d0ee252 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 18:00:14 +0200 Subject: [PATCH 089/132] fix file name --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0093e00..1fc303b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -162,11 +162,11 @@ jobs: summary_url: ${{ env.summary-url }} run: | mkdir -p /tmp/comment-json - json_filename=$( \ + json_filename="$( \ echo "report-comment-json-merge-$full_image_name-$config_name" | \ tr '/:._' '-' \ - ) - echo "json-filename=${json_filename}.json" >> "$GITHUB_OUTPUT" + ).json" + echo "json-filename=$json_filename" >> "$GITHUB_OUTPUT" json_file="/tmp/comment-json/$json_filename" jq \ -n \ From eb603871ff53b072d6bf12d05556797b3005dc17 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Thu, 30 May 2024 18:07:50 +0200 Subject: [PATCH 090/132] Add the double `/report` --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1fc303b..d9fcbb7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -149,7 +149,7 @@ jobs: --source "$REPORT_DIR" \ --destination "$unique_directory" \ --recursive - echo "report-url=${unique_directory}/index.html" >> "$GITHUB_OUTPUT" + echo "report-url=${unique_directory}/report/report/index.html" >> "$GITHUB_OUTPUT" - name: Prepare comment artifact id: prepare-artifact From 337a7d367665dbd1d2a12a536d02103d1d4c4e22 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 12:36:51 +0200 Subject: [PATCH 091/132] Specify analysuis outputs as config --- R/run_pacta_reporting_process.R | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index fe16306..531f227 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,5 +1,6 @@ run_pacta_reporting_process <- function( - raw_params = commandArgs(trailingOnly = TRUE) + raw_params = commandArgs(trailingOnly = TRUE), + analysis_outputs_dir = Sys.getenv("ANALYSIS_OUTPUTS_DIR") ) { # defaulting to WARN to maintain current (silent) behavior. @@ -22,7 +23,7 @@ run_pacta_reporting_process <- function( # quit if there's no relevant PACTA assets ------------------------------------- log_debug("Checking for PACTA relevant data in portfolio results.") - total_portfolio_path <- file.path(cfg$output_dir, "total_portfolio.rds") + total_portfolio_path <- file.path(analysis_outputs_dir, "total_portfolio.rds") if (file.exists(total_portfolio_path)) { total_portfolio <- readRDS(total_portfolio_path) log_trace("Checking for PACTA relevant data in file: \"{total_portfolio_path}\".") @@ -49,7 +50,7 @@ run_pacta_reporting_process <- function( log_debug("Loading audit file.") audit_file <- read_rds_or_return_alt_data( - filepath = file.path(cfg$output_dir, "audit_file.rds"), + filepath = file.path(analysis_outputs_dir, "audit_file.rds"), alt_return = pacta.portfolio.utils::empty_audit_file() ) audit_file <- add_inv_and_port_names_if_needed( @@ -60,7 +61,7 @@ run_pacta_reporting_process <- function( log_debug("Loading portfolio overview.") portfolio_overview <- read_rds_or_return_alt_data( - filepath = file.path(cfg$output_dir, "overview_portfolio.rds"), + filepath = file.path(analysis_outputs_dir, "overview_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_overview() ) portfolio_overview <- add_inv_and_port_names_if_needed( @@ -71,7 +72,7 @@ run_pacta_reporting_process <- function( log_debug("Loading emissions.") emissions <- read_rds_or_return_alt_data( - filepath = file.path(cfg$output_dir, "emissions.rds"), + filepath = file.path(analysis_outputs_dir, "emissions.rds"), alt_return = pacta.portfolio.utils::empty_emissions_results() ) emissions <- add_inv_and_port_names_if_needed( @@ -82,7 +83,7 @@ run_pacta_reporting_process <- function( log_debug("Loading total portfolio results.") total_portfolio <- read_rds_or_return_alt_data( - filepath = file.path(cfg$output_dir, "total_portfolio.rds"), + filepath = file.path(analysis_outputs_dir, "total_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) total_portfolio <- add_inv_and_port_names_if_needed( @@ -93,7 +94,7 @@ run_pacta_reporting_process <- function( log_debug("Loading portfolio equity results.") equity_results_portfolio <- read_rds_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Equity_results_portfolio.rds"), + filepath = file.path(analysis_outputs_dir, "Equity_results_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) equity_results_portfolio <- add_inv_and_port_names_if_needed( @@ -104,7 +105,7 @@ run_pacta_reporting_process <- function( log_debug("Loading portfolio bonds results.") bonds_results_portfolio <- read_rds_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Bonds_results_portfolio.rds"), + filepath = file.path(analysis_outputs_dir, "Bonds_results_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) bonds_results_portfolio <- add_inv_and_port_names_if_needed( @@ -115,7 +116,7 @@ run_pacta_reporting_process <- function( log_debug("Loading company equity results.") equity_results_company <- read_rds_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Equity_results_company.rds"), + filepath = file.path(analysis_outputs_dir, "Equity_results_company.rds"), alt_return = pacta.portfolio.utils::empty_company_results() ) equity_results_company <- add_inv_and_port_names_if_needed( @@ -126,7 +127,7 @@ run_pacta_reporting_process <- function( log_debug("Loading company bonds results.") bonds_results_company <- read_rds_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Bonds_results_company.rds"), + filepath = file.path(analysis_outputs_dir, "Bonds_results_company.rds"), alt_return = pacta.portfolio.utils::empty_company_results() ) bonds_results_company <- add_inv_and_port_names_if_needed( @@ -137,7 +138,7 @@ run_pacta_reporting_process <- function( log_debug("Loading equity map results.") equity_results_map <- read_rds_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Equity_results_map.rds"), + filepath = file.path(analysis_outputs_dir, "Equity_results_map.rds"), alt_return = pacta.portfolio.utils::empty_map_results() ) equity_results_map <- add_inv_and_port_names_if_needed( @@ -148,7 +149,7 @@ run_pacta_reporting_process <- function( log_debug("Loading bonds map results.") bonds_results_map <- read_rds_or_return_alt_data( - filepath = file.path(cfg$output_dir, "Bonds_results_map.rds"), + filepath = file.path(analysis_outputs_dir, "Bonds_results_map.rds"), alt_return = pacta.portfolio.utils::empty_map_results() ) bonds_results_map <- add_inv_and_port_names_if_needed( From 2de9cd309736f04fbef5d027e423314dc164b793 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 12:44:10 +0200 Subject: [PATCH 092/132] Add benchmarks data to config --- R/run_pacta_reporting_process.R | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 531f227..72b9881 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,6 +1,9 @@ run_pacta_reporting_process <- function( raw_params = commandArgs(trailingOnly = TRUE), - analysis_outputs_dir = Sys.getenv("ANALYSIS_OUTPUTS_DIR") + analysis_outputs_dir = Sys.getenv("ANALYSIS_OUTPUTS_DIR"), + benchmarks_dir = Sys.getenv("BENCHMARKS_DIR"), + report_output_dir = Sys.getenv("REPORT_OUTPUT_DIR"), + summary_output_dir = Sys.getenv("SUMMARY_OUTPUT_DIR") ) { # defaulting to WARN to maintain current (silent) behavior. @@ -160,33 +163,33 @@ run_pacta_reporting_process <- function( log_debug("Loading portfolio equity peer results.") peers_equity_results_portfolio <- read_rds_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio.rds")), + filepath = file.path(benchmarks_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading portfolio bonds peer results.") peers_bonds_results_portfolio <- read_rds_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio.rds")), + filepath = file.path(benchmarks_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading index equity peer results.") peers_equity_results_user <- read_rds_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio_ind.rds")), + filepath = file.path(benchmarks_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio_ind.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading index bonds peer results.") peers_bonds_results_user <- read_rds_or_return_alt_data( - filepath = file.path(cfg$data_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio_ind.rds")), + filepath = file.path(benchmarks_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio_ind.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading index equity portfolio results.") - indices_equity_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_equity_results_portfolio.rds")) + indices_equity_results_portfolio <- readRDS(file.path(benchmarks_dir, "Indices_equity_results_portfolio.rds")) log_debug("Loading index bonds portfolio results.") - indices_bonds_results_portfolio <- readRDS(file.path(cfg$data_dir, "Indices_bonds_results_portfolio.rds")) + indices_bonds_results_portfolio <- readRDS(file.path(benchmarks_dir, "Indices_bonds_results_portfolio.rds")) # create interactive report ---------------------------------------------------- From a203a85e3d148cc5ea4c8be7c7fc93a806167460 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 13:04:27 +0200 Subject: [PATCH 093/132] Change configs --- NAMESPACE | 1 + R/run_pacta_reporting_process.R | 41 ++++++++++++++++++++++--------- R/utils.R | 11 +++++++-- R/workflow.pacta.report-package.R | 1 + docker-compose.yml | 25 +++++++++++-------- 5 files changed, 55 insertions(+), 24 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 38ac726..0a6e867 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,6 +1,7 @@ # Generated by roxygen2: do not edit by hand importFrom(logger,log_debug) +importFrom(logger,log_error) importFrom(logger,log_fatal) importFrom(logger,log_info) importFrom(logger,log_trace) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 72b9881..74b64fd 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,10 +1,27 @@ run_pacta_reporting_process <- function( raw_params = commandArgs(trailingOnly = TRUE), - analysis_outputs_dir = Sys.getenv("ANALYSIS_OUTPUTS_DIR"), + analysis_output_dir = Sys.getenv("ANALYSIS_OUTPUTS_DIR"), benchmarks_dir = Sys.getenv("BENCHMARKS_DIR"), report_output_dir = Sys.getenv("REPORT_OUTPUT_DIR"), summary_output_dir = Sys.getenv("SUMMARY_OUTPUT_DIR") ) { + log_debug("Checking configuration.") + if (is.null(analysis_output_dir) || analysis_output_dir == "") { + log_error("ANALYSIS_OUTPUTS_DIR not set.") + stop("ANALYSIS_OUTPUTS_DIR not set.") + } + if (is.null(benchmarks_dir) || benchmarks_dir == "") { + log_error("BENCHMARKS_DIR not set.") + stop("BENCHMARKS_DIR not set.") + } + if (is.null(report_output_dir) || report_output_dir == "") { + log_error("REPORT_OUTPUT_DIR not set.") + stop("REPORT_OUTPUT_DIR not set.") + } + if (is.null(summary_output_dir) || summary_output_dir == "") { + log_error("SUMMARY_OUTPUT_DIR not set.") + stop("SUMMARY_OUTPUT_DIR not set.") + } # defaulting to WARN to maintain current (silent) behavior. logger::log_threshold(Sys.getenv("LOG_LEVEL", "INFO")) @@ -26,7 +43,7 @@ run_pacta_reporting_process <- function( # quit if there's no relevant PACTA assets ------------------------------------- log_debug("Checking for PACTA relevant data in portfolio results.") - total_portfolio_path <- file.path(analysis_outputs_dir, "total_portfolio.rds") + total_portfolio_path <- file.path(analysis_output_dir, "total_portfolio.rds") if (file.exists(total_portfolio_path)) { total_portfolio <- readRDS(total_portfolio_path) log_trace("Checking for PACTA relevant data in file: \"{total_portfolio_path}\".") @@ -53,7 +70,7 @@ run_pacta_reporting_process <- function( log_debug("Loading audit file.") audit_file <- read_rds_or_return_alt_data( - filepath = file.path(analysis_outputs_dir, "audit_file.rds"), + filepath = file.path(analysis_output_dir, "audit_file.rds"), alt_return = pacta.portfolio.utils::empty_audit_file() ) audit_file <- add_inv_and_port_names_if_needed( @@ -64,7 +81,7 @@ run_pacta_reporting_process <- function( log_debug("Loading portfolio overview.") portfolio_overview <- read_rds_or_return_alt_data( - filepath = file.path(analysis_outputs_dir, "overview_portfolio.rds"), + filepath = file.path(analysis_output_dir, "overview_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_overview() ) portfolio_overview <- add_inv_and_port_names_if_needed( @@ -75,7 +92,7 @@ run_pacta_reporting_process <- function( log_debug("Loading emissions.") emissions <- read_rds_or_return_alt_data( - filepath = file.path(analysis_outputs_dir, "emissions.rds"), + filepath = file.path(analysis_output_dir, "emissions.rds"), alt_return = pacta.portfolio.utils::empty_emissions_results() ) emissions <- add_inv_and_port_names_if_needed( @@ -86,7 +103,7 @@ run_pacta_reporting_process <- function( log_debug("Loading total portfolio results.") total_portfolio <- read_rds_or_return_alt_data( - filepath = file.path(analysis_outputs_dir, "total_portfolio.rds"), + filepath = file.path(analysis_output_dir, "total_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) total_portfolio <- add_inv_and_port_names_if_needed( @@ -97,7 +114,7 @@ run_pacta_reporting_process <- function( log_debug("Loading portfolio equity results.") equity_results_portfolio <- read_rds_or_return_alt_data( - filepath = file.path(analysis_outputs_dir, "Equity_results_portfolio.rds"), + filepath = file.path(analysis_output_dir, "Equity_results_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) equity_results_portfolio <- add_inv_and_port_names_if_needed( @@ -108,7 +125,7 @@ run_pacta_reporting_process <- function( log_debug("Loading portfolio bonds results.") bonds_results_portfolio <- read_rds_or_return_alt_data( - filepath = file.path(analysis_outputs_dir, "Bonds_results_portfolio.rds"), + filepath = file.path(analysis_output_dir, "Bonds_results_portfolio.rds"), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) bonds_results_portfolio <- add_inv_and_port_names_if_needed( @@ -119,7 +136,7 @@ run_pacta_reporting_process <- function( log_debug("Loading company equity results.") equity_results_company <- read_rds_or_return_alt_data( - filepath = file.path(analysis_outputs_dir, "Equity_results_company.rds"), + filepath = file.path(analysis_output_dir, "Equity_results_company.rds"), alt_return = pacta.portfolio.utils::empty_company_results() ) equity_results_company <- add_inv_and_port_names_if_needed( @@ -130,7 +147,7 @@ run_pacta_reporting_process <- function( log_debug("Loading company bonds results.") bonds_results_company <- read_rds_or_return_alt_data( - filepath = file.path(analysis_outputs_dir, "Bonds_results_company.rds"), + filepath = file.path(analysis_output_dir, "Bonds_results_company.rds"), alt_return = pacta.portfolio.utils::empty_company_results() ) bonds_results_company <- add_inv_and_port_names_if_needed( @@ -141,7 +158,7 @@ run_pacta_reporting_process <- function( log_debug("Loading equity map results.") equity_results_map <- read_rds_or_return_alt_data( - filepath = file.path(analysis_outputs_dir, "Equity_results_map.rds"), + filepath = file.path(analysis_output_dir, "Equity_results_map.rds"), alt_return = pacta.portfolio.utils::empty_map_results() ) equity_results_map <- add_inv_and_port_names_if_needed( @@ -152,7 +169,7 @@ run_pacta_reporting_process <- function( log_debug("Loading bonds map results.") bonds_results_map <- read_rds_or_return_alt_data( - filepath = file.path(analysis_outputs_dir, "Bonds_results_map.rds"), + filepath = file.path(analysis_output_dir, "Bonds_results_map.rds"), alt_return = pacta.portfolio.utils::empty_map_results() ) bonds_results_map <- add_inv_and_port_names_if_needed( diff --git a/R/utils.R b/R/utils.R index 9bd9426..a602a3a 100644 --- a/R/utils.R +++ b/R/utils.R @@ -1,8 +1,13 @@ read_rds_or_return_alt_data <- function(filepath, alt_return = NULL) { + log_trace("Reading RDS file: \"filepath\"") if (file.exists(filepath)) { - return(readRDS(filepath)) + log_trace("File found. Reading data.") + data <- readRDS(filepath) + } else { + log_warn("File not found. Returning alternative data.") + data <- alt_return } - alt_return + return(data) } add_inv_and_port_names_if_needed <- function(data, portfolio_name, investor_name) { @@ -11,10 +16,12 @@ add_inv_and_port_names_if_needed <- function(data, portfolio_name, investor_name } if (!"portfolio_name" %in% names(data)) { + log_trace("Adding portfolio_name column to data.") data <- dplyr::mutate(data, portfolio_name = portfolio_name, .before = dplyr::everything()) } if (!"investor_name" %in% names(data)) { + log_trace("Adding investor_name column to data.") data <- dplyr::mutate(data, investor_name = investor_name, .before = dplyr::everything()) } diff --git a/R/workflow.pacta.report-package.R b/R/workflow.pacta.report-package.R index a270edb..98a01ae 100644 --- a/R/workflow.pacta.report-package.R +++ b/R/workflow.pacta.report-package.R @@ -3,6 +3,7 @@ ## usethis namespace: start #' @importFrom logger log_debug +#' @importFrom logger log_error #' @importFrom logger log_fatal #' @importFrom logger log_info #' @importFrom logger log_trace diff --git a/docker-compose.yml b/docker-compose.yml index f2c4e9e..5171c70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,19 +10,24 @@ services: # command: "/workflow.pacta.report/input_dir/default_config.json" environment: LOG_LEVEL: TRACE - PACTA_DATA_DIR: "/mnt/pacta-data" - OUTPUT_DIR: "/mnt/output_dir" - PORTFOLIO_DIR: "/mnt/portfolios" + ANALYSIS_OUTPUTS_DIR: "/mnt/analysis_output_dir" + BENCHMARKS_DIR: "/mnt/benchmarks_dir" + REPORT_OUTPUT_DIR: "/mnt/report_output_dir" + SUMMARY_OUTPUT_DIR: "/mnt/summary_output_dir" volumes: - type: bind source: ${data_dir:-./pacta-data} target: /mnt/pacta-data/ read_only: true - type: bind - source: ${output_dir:-./output_dir} - target: /mnt/output_dir - read_only: false - # - type: bind - # source: ${input_dir:-./portfolios} - # target: /mnt/portfolios - # read_only: true + source: ${analysis_output_dir:-./analysis_output_dir} + target: /mnt/analysis_output_dir + read_only: true + - type: bind + source: ${report_output_dir:-./report_outputs_dir} + target: /mnt/report_output_dir + read_only: true + - type: bind + source: ${summary_output_dir:-./summary_outputs_dir} + target: /mnt/summary_output_dir + read_only: true From 4367209b71ea9f41075e452a231bf7b583015250 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 15:09:28 +0200 Subject: [PATCH 094/132] Isolate Report Output dir --- R/prepare_interactive_report.R | 13 +++++-------- R/run_pacta_reporting_process.R | 7 ++++++- docker-compose.yml | 12 ++++++------ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/R/prepare_interactive_report.R b/R/prepare_interactive_report.R index ae9f0ab..b29ba1b 100644 --- a/R/prepare_interactive_report.R +++ b/R/prepare_interactive_report.R @@ -1,5 +1,6 @@ prepare_interactive_report <- function( cfg, + report_output_dir, audit_file, emissions, portfolio_overview, @@ -14,7 +15,8 @@ prepare_interactive_report <- function( peers_bonds_results_portfolio, peers_bonds_results_user, peers_equity_results_portfolio, - peers_equity_results_user + peers_equity_results_user, + analysis_output_manifest ) { log_debug("Preparing to create interactive report.") @@ -43,16 +45,11 @@ prepare_interactive_report <- function( # combine config files to send to create_interactive_report() log_trace("Defining configs and manifest.") - pacta_data_public_manifest <- - list( - creation_time_date = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$creation_time_date, - outputs_manifest = jsonlite::read_json(file.path(cfg$data_dir, "manifest.json"))$outputs_manifest - ) configs <- list( portfolio_config = cfg, - pacta_data_public_manifest = pacta_data_public_manifest + analysis_output_manifest = analysis_output_manifest ) # workaround a bug in {config} v0.3.2 that only adds "config" class to objects it creates @@ -66,7 +63,7 @@ prepare_interactive_report <- function( log_info("Creating interactive report.") pacta.portfolio.report::create_interactive_report( template_dir = template_dir, - output_dir = cfg$report_dir, + output_dir = report_output_dir, survey_dir = cfg$survey_dir, real_estate_dir = cfg$real_estate_dir, language_select = cfg$language_select, diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 74b64fd..b985c98 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -178,6 +178,8 @@ run_pacta_reporting_process <- function( investor_name = cfg[["investor_name"]] ) + analysis_output_manifest <- jsonlite::read_json(file.path(analysis_output_dir, "manifest.json")) + log_debug("Loading portfolio equity peer results.") peers_equity_results_portfolio <- read_rds_or_return_alt_data( filepath = file.path(benchmarks_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio.rds")), @@ -212,6 +214,7 @@ run_pacta_reporting_process <- function( prepare_interactive_report( cfg = cfg, + report_output_dir = report_output_dir, audit_file = audit_file, emissions = emissions, portfolio_overview = portfolio_overview, @@ -226,12 +229,14 @@ run_pacta_reporting_process <- function( peers_bonds_results_portfolio = peers_bonds_results_portfolio, peers_bonds_results_user = peers_bonds_results_user, peers_equity_results_portfolio = peers_equity_results_portfolio, - peers_equity_results_user = peers_equity_results_user + peers_equity_results_user = peers_equity_results_user, + analysis_output_manifest = analysis_output_manifest ) # create executive summary ----------------------------------------------------- prepare_executive_summary( cfg = cfg, + summary_output_dir = summary_output_dir, total_portfolio = total_portfolio, equity_results_portfolio = equity_results_portfolio, bonds_results_portfolio = bonds_results_portfolio, diff --git a/docker-compose.yml b/docker-compose.yml index 5171c70..e58de31 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,18 +16,18 @@ services: SUMMARY_OUTPUT_DIR: "/mnt/summary_output_dir" volumes: - type: bind - source: ${data_dir:-./pacta-data} - target: /mnt/pacta-data/ + source: ${benchmarks_dir:-./benchmarks_dir} + target: /mnt/benchmarks_dir/ read_only: true - type: bind source: ${analysis_output_dir:-./analysis_output_dir} target: /mnt/analysis_output_dir read_only: true - type: bind - source: ${report_output_dir:-./report_outputs_dir} + source: ${report_output_dir:-./report_output_dir} target: /mnt/report_output_dir - read_only: true + read_only: false - type: bind - source: ${summary_output_dir:-./summary_outputs_dir} + source: ${summary_output_dir:-./summary_output_dir} target: /mnt/summary_output_dir - read_only: true + read_only: false From f1ea7314a16e8e37cb9b0dd7d5b1ce19ec62a91c Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 15:13:57 +0200 Subject: [PATCH 095/132] Isolate summary outputs --- R/prepare_executive_summary.R | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/R/prepare_executive_summary.R b/R/prepare_executive_summary.R index 5ce2aa9..53c7b87 100644 --- a/R/prepare_executive_summary.R +++ b/R/prepare_executive_summary.R @@ -1,5 +1,6 @@ prepare_executive_summary <- function( cfg, + summary_output_dir, total_portfolio, equity_results_portfolio, bonds_results_portfolio, @@ -17,7 +18,7 @@ prepare_executive_summary <- function( survey_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "survey")) real_estate_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "real_estate")) score_card_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "score_card")) - es_dir <- file.path(cfg$output_dir, "executive_summary") + es_dir <- file.path(summary_output_dir, "executive_summary") if (!dir.exists(es_dir)) { dir.create(es_dir, showWarnings = FALSE, recursive = TRUE) } @@ -28,8 +29,8 @@ prepare_executive_summary <- function( "extdata", exec_summary_template_name, package = "pacta.executive.summary" ) - invisible(file.copy(exec_summary_builtin_template_path, cfg$output_dir, recursive = TRUE, copy.mode = FALSE)) - exec_summary_template_path <- file.path(cfg$output_dir, exec_summary_template_name) + invisible(file.copy(exec_summary_builtin_template_path, summary_output_dir, recursive = TRUE, copy.mode = FALSE)) + exec_summary_template_path <- file.path(summary_output_dir, exec_summary_template_name) if ( dir.exists(exec_summary_template_path) && ( @@ -94,7 +95,7 @@ prepare_executive_summary <- function( total_portfolio = total_portfolio, scenario_selected = "1.5C-Unif", currency_exchange_value = cfg$currency_exchange_value, - log_dir = cfg$output_dir + log_dir = summary_output_dir ) } else { # this is required for the online tool to know that the process has been completed. From ebeb461a1c87aab2cabf1c878f0910ca526a9223 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 15:23:53 +0200 Subject: [PATCH 096/132] Update testing script --- .github/workflows/test.yml | 13 +++++++++---- R/run_pacta_reporting_process.R | 2 +- docker-compose.yml | 2 +- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d9fcbb7..26a25bc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -119,10 +119,15 @@ jobs: chmod -R 777 "$REPORT_DIR" docker run \ --network none \ - --env LOG_LEVEL=DEBUG \ - --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \ - --mount type=bind,source=${WORKSPACE}/${OUTPUT_DIR},target=/mnt/output_dir \ - --mount type=bind,source=${WORKSPACE}/${REPORT_DIR},target=/mnt/report_dir \ + --env LOG_LEVEL=TRACE \ + --env ANALYSIS_OUTPUT_DIR="/mnt/analysis_output_dir" \ + --env BENCHMARKS_DIR="/mnt/benchmarks_dir" \ + --env REPORT_OUTPUT_DIR="/mnt/report_output_dir" \ + --env SUMMARY_OUTPUT_DIR="/mnt/summary_output_dir" \ + --mount type=bind,readonly,source=${WORKSPACE}/${BENCHMARKS_DIR},target=/mnt/benchmarks_dir \ + --mount type=bind,readonly,source=${WORKSPACE}/${ANALYSIS_OUTPUT_DIR},target=/mnt/analysis_output_dir \ + --mount type=bind,readonly,source=${WORKSPACE}/${REPORT_OUTPUT_DIR},target=/mnt/report_output_dir \ + --mount type=bind,readonly,source=${WORKSPACE}/${SUMMARY_OUTPUT_DIR},target=/mnt/summary_output_dir \ $FULL_IMAGE_NAME # $FULL_IMAGE_NAME \ # $PARAMETERS diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index b985c98..fc135bc 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -1,6 +1,6 @@ run_pacta_reporting_process <- function( raw_params = commandArgs(trailingOnly = TRUE), - analysis_output_dir = Sys.getenv("ANALYSIS_OUTPUTS_DIR"), + analysis_output_dir = Sys.getenv("ANALYSIS_OUTPUT_DIR"), benchmarks_dir = Sys.getenv("BENCHMARKS_DIR"), report_output_dir = Sys.getenv("REPORT_OUTPUT_DIR"), summary_output_dir = Sys.getenv("SUMMARY_OUTPUT_DIR") diff --git a/docker-compose.yml b/docker-compose.yml index e58de31..8dcb590 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: # command: "/workflow.pacta.report/input_dir/default_config.json" environment: LOG_LEVEL: TRACE - ANALYSIS_OUTPUTS_DIR: "/mnt/analysis_output_dir" + ANALYSIS_OUTPUT_DIR: "/mnt/analysis_output_dir" BENCHMARKS_DIR: "/mnt/benchmarks_dir" REPORT_OUTPUT_DIR: "/mnt/report_output_dir" SUMMARY_OUTPUT_DIR: "/mnt/summary_output_dir" From 330471cfaad903fa57ed82b3d124d0c3af782b76 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 15:26:38 +0200 Subject: [PATCH 097/132] Update test prep --- .github/workflows/test.yml | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 26a25bc..d7960e3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,20 +64,6 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version - - name: Download pacta-data - uses: azure/CLI@v2 - env: - PACTA_DATA_URL: ${{ steps.prepare.outputs.pacta-data-url }} - with: - # azcliversion: 2.30.0 - inlineScript: | - az storage copy \ - --source "$PACTA_DATA_URL/manifest.json" \ - --destination "pacta-data/manifest.json" \ - --recursive \ - --exclude-pattern "*.sqlite" - # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version - name: Download outputs from workflow.pacta uses: azure/CLI@v2 @@ -88,7 +74,7 @@ jobs: inlineScript: | az storage copy \ --source "$RESULTS_URL/*" \ - --destination "output_dir/" \ + --destination "analysis_output_dir/" \ --recursive \ --exclude-pattern "*.sqlite" @@ -102,16 +88,17 @@ jobs: inlineScript: | az storage copy \ --source "$BENCHMARKS_URL/*" \ - --destination "pacta-data" \ + --destination "benchmarks_dir/" \ --recursive \ --exclude-pattern "*.sqlite" - name: Run Docker Image env: FULL_IMAGE_NAME: ${{ inputs.full-image-name }} - OUTPUT_DIR: output_dir - REPORT_DIR: report - PACTA_DATA_DIR: pacta-data + ANALYSIS_OUTPUT_DIR: analysis_output_dir + BENCHMARKS_DIR: benchmarks_dir + REPORT_OUTPUT_DIR: report_output_dir + SUMMARY_OUTPUT_DIR: summary_output_dir WORKSPACE: ${{ github.workspace }} PARAMETERS: ${{ steps.prepare.outputs.parameters }} run: | @@ -134,7 +121,8 @@ jobs: - name: List outputs run: | - ls -lR output_dir + ls -lR report_output_dir + ls -lR summary_output_dir # https://github.com/marketplace/actions/azure-cli-action#workflow-to-execute-an-azure-cli-script-of-a-specific-cli-version - name: Upload results to blob store From fbe17a755e0efbc7435eb1544f9348279f0940b0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 15:32:18 +0200 Subject: [PATCH 098/132] Create and change permissions for correct dir --- .github/workflows/test.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7960e3..d0b19f7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -102,8 +102,10 @@ jobs: WORKSPACE: ${{ github.workspace }} PARAMETERS: ${{ steps.prepare.outputs.parameters }} run: | - mkdir -p "$REPORT_DIR" - chmod -R 777 "$REPORT_DIR" + mkdir -p "$REPORT_OUTPUT_DIR" + chmod -R 777 "$REPORT_OUTPUT_DIR" + mkdir -p "$SUMMARY_OUTPUT_DIR" + chmod -R 777 "$SUMMARY_OUTPUT_DIR" docker run \ --network none \ --env LOG_LEVEL=TRACE \ From 109613651f0f7180b9ddb25d31609d6f66e1c70a Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 15:34:37 +0200 Subject: [PATCH 099/132] Upload correct files --- .github/workflows/test.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d0b19f7..784b54f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -135,16 +135,23 @@ jobs: GITHUB_REF_NAME: ${{ github.ref_name}} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} GITHUB_RUN_NUMBER: ${{ github.run_number }} - REPORT_DIR: report + REPORT_OUTPUT_DIR: report_output_dir + SUMMARY_OUTPUT_DIR: summary_output_dir RESULTS_URL: ${{ inputs.results-url }} with: inlineScript: | unique_directory="$RESULTS_URL/$GITHUB_REF_NAME/$GITHUB_RUN_NUMBER/$GITHUB_RUN_ATTEMPT/$CONFIG_NAME" az storage copy \ - --source "$REPORT_DIR" \ + --source "$REPORT_OUTPUT_DIR" \ + --destination "$unique_directory" \ + --recursive + echo "report-url=${unique_directory}/report/index.html" >> "$GITHUB_OUTPUT" + + az storage copy \ + --source "$SUMMARY_OUTPUT_DIR" \ --destination "$unique_directory" \ --recursive - echo "report-url=${unique_directory}/report/report/index.html" >> "$GITHUB_OUTPUT" + echo "summary-url=${unique_directory}/executive_summary/" >> "$GITHUB_OUTPUT" - name: Prepare comment artifact id: prepare-artifact From 741224188cd336ae2b27b691f9b0230798962f3c Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 15:35:57 +0200 Subject: [PATCH 100/132] Provide correct link to ES --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 784b54f..06b4eca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -151,7 +151,8 @@ jobs: --source "$SUMMARY_OUTPUT_DIR" \ --destination "$unique_directory" \ --recursive - echo "summary-url=${unique_directory}/executive_summary/" >> "$GITHUB_OUTPUT" + summary_files="$(ls $SUMMARY_OUTPUT_DIR/executive_summary/*.pdf)" + echo "summary-url=${unique_directory}/executive_summary/$summary_files" >> "$GITHUB_OUTPUT" - name: Prepare comment artifact id: prepare-artifact From 1410b5b2bda35772245848209746f105a32761a6 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 15:40:50 +0200 Subject: [PATCH 101/132] remove readonly flags on output binds --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 06b4eca..4011de8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -115,8 +115,8 @@ jobs: --env SUMMARY_OUTPUT_DIR="/mnt/summary_output_dir" \ --mount type=bind,readonly,source=${WORKSPACE}/${BENCHMARKS_DIR},target=/mnt/benchmarks_dir \ --mount type=bind,readonly,source=${WORKSPACE}/${ANALYSIS_OUTPUT_DIR},target=/mnt/analysis_output_dir \ - --mount type=bind,readonly,source=${WORKSPACE}/${REPORT_OUTPUT_DIR},target=/mnt/report_output_dir \ - --mount type=bind,readonly,source=${WORKSPACE}/${SUMMARY_OUTPUT_DIR},target=/mnt/summary_output_dir \ + --mount type=bind,source=${WORKSPACE}/${REPORT_OUTPUT_DIR},target=/mnt/report_output_dir \ + --mount type=bind,source=${WORKSPACE}/${SUMMARY_OUTPUT_DIR},target=/mnt/summary_output_dir \ $FULL_IMAGE_NAME # $FULL_IMAGE_NAME \ # $PARAMETERS From ba0bc932b76f9254f90de7f2b2bf8db7eef8cd5e Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 15:46:36 +0200 Subject: [PATCH 102/132] update summary url reference --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4011de8..99f4cda 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -162,7 +162,7 @@ jobs: full_image_name: ${{ inputs.full-image-name }} git_sha: ${{ github.event.pull_request.head.sha }} report_url: ${{ steps.upload.outputs.report-url }} - summary_url: ${{ env.summary-url }} + summary_url: ${{ steps.upload.outputs.summary-url }} run: | mkdir -p /tmp/comment-json json_filename="$( \ From 1b0726a6205979d303673519bf31aab8994c8359 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 16:09:30 +0200 Subject: [PATCH 103/132] Add dependency on `pacta.workflow.utils` --- DESCRIPTION | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 75d2088..b38e968 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -28,8 +28,10 @@ Imports: pacta.executive.summary, pacta.portfolio.report, pacta.portfolio.utils, + pacta.workflow.utils, readr Remotes: RMI-PACTA/pacta.executive.summary, RMI-PACTA/pacta.portfolio.report@issues-found-in-workflow.pacta.report, - RMI-PACTA/pacta.portfolio.utils + RMI-PACTA/pacta.portfolio.utils, + RMI-PACTA/pacta.workflow.utils From 07c2a6bf83cea755c10d5b47a01aba02e3781911 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 16:22:15 +0200 Subject: [PATCH 104/132] read params using `pacta.workflow.utils` --- R/run_pacta_reporting_process.R | 44 +++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index fc135bc..c163d23 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -31,14 +31,44 @@ run_pacta_reporting_process <- function( log_info("Starting portfolio report process") - log_trace("Determining configuration file path") - cfg_path <- commandArgs(trailingOnly = TRUE) - if (length(cfg_path) == 0 || cfg_path == "") { - log_warn("No configuration file specified, using default") - cfg_path <- "/workflow.pacta.report/input_dir/default_config.json" + # Read Params + log_trace("Processing input parameters.") + if (length(raw_params) == 0L || all(raw_params == "")) { + log_error("No parameters specified.") } - log_debug("Loading configuration from file: \"{cfg_path}\".") - cfg <- jsonlite::fromJSON(cfg_path) + + # log_trace("Validating raw input parameters.") + # raw_input_validation_results <- jsonvalidate::json_validate( + # json = raw_params, + # schema = system.file( + # "extdata", "schema", "rawParameters.json", + # package = "workflow.pacta.report" + # ), + # verbose = TRUE, + # greedy = FALSE, + # engine = "ajv" + # ) + # if (raw_input_validation_results) { + # log_trace("Raw input parameters are valid.") + # } else { + # log_error( + # "Invalid raw input parameters. ", + # "Must include \"inherit\" key, or match full schema." + # ) + # stop("Invalid raw input parameters.") + # } + + params <- pacta.workflow.utils:::parse_params( + json = raw_params, + inheritence_search_paths = system.file( + "extdata", "parameters", + package = "workflow.pacta.report" + ) #, + # schema_file = system.file( + # "extdata", "schema", "portfolioParameters_0-0-1.json", + # package = "workflow.pacta.report" + # ) + ) # quit if there's no relevant PACTA assets ------------------------------------- From a26160fa32795491a4451e33488264f9fb2d7524 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 16:25:25 +0200 Subject: [PATCH 105/132] use params object for language select --- R/run_pacta_reporting_process.R | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index c163d23..416f016 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -86,11 +86,14 @@ run_pacta_reporting_process <- function( # fix parameters --------------------------------------------------------------- - if (cfg$project_code == "GENERAL") { + if ( + params[["project_code"]] == "GENERAL" && + params[["language_select"]] != "EN" + ) { log_warn("Overriding language selection to \"EN\" for \"GENERAL\".") - cfg$language_select <- "EN" + params[["language_select"]] <- "EN" } else { - log_trace("Using language selection: \"{cfg$language_select}\".") + log_trace("Using language selection: \"{params[["language_select"]]}\".") } @@ -212,25 +215,25 @@ run_pacta_reporting_process <- function( log_debug("Loading portfolio equity peer results.") peers_equity_results_portfolio <- read_rds_or_return_alt_data( - filepath = file.path(benchmarks_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio.rds")), + filepath = file.path(benchmarks_dir, paste0(params[["project_code"]], "_peers_equity_results_portfolio.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading portfolio bonds peer results.") peers_bonds_results_portfolio <- read_rds_or_return_alt_data( - filepath = file.path(benchmarks_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio.rds")), + filepath = file.path(benchmarks_dir, paste0(params[["project_code"]], "_peers_bonds_results_portfolio.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading index equity peer results.") peers_equity_results_user <- read_rds_or_return_alt_data( - filepath = file.path(benchmarks_dir, paste0(cfg$project_code, "_peers_equity_results_portfolio_ind.rds")), + filepath = file.path(benchmarks_dir, paste0(params[["project_code"]], "_peers_equity_results_portfolio_ind.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) log_debug("Loading index bonds peer results.") peers_bonds_results_user <- read_rds_or_return_alt_data( - filepath = file.path(benchmarks_dir, paste0(cfg$project_code, "_peers_bonds_results_portfolio_ind.rds")), + filepath = file.path(benchmarks_dir, paste0(params[["project_code"]], "_peers_bonds_results_portfolio_ind.rds")), alt_return = pacta.portfolio.utils::empty_portfolio_results() ) From be4eb41a4fd37879dd67aeb438b33f8774741914 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 16:26:12 +0200 Subject: [PATCH 106/132] rename `cfg[[]]` to `params[[]]` --- R/run_pacta_reporting_process.R | 40 ++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 416f016..8d75e0c 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -108,8 +108,8 @@ run_pacta_reporting_process <- function( ) audit_file <- add_inv_and_port_names_if_needed( data = audit_file, - portfolio_name = cfg[["portfolio_name"]], - investor_name = cfg[["investor_name"]] + portfolio_name = params[["portfolio_name"]], + investor_name = params[["investor_name"]] ) log_debug("Loading portfolio overview.") @@ -119,8 +119,8 @@ run_pacta_reporting_process <- function( ) portfolio_overview <- add_inv_and_port_names_if_needed( data = portfolio_overview, - portfolio_name = cfg[["portfolio_name"]], - investor_name = cfg[["investor_name"]] + portfolio_name = params[["portfolio_name"]], + investor_name = params[["investor_name"]] ) log_debug("Loading emissions.") @@ -130,8 +130,8 @@ run_pacta_reporting_process <- function( ) emissions <- add_inv_and_port_names_if_needed( data = emissions, - portfolio_name = cfg[["portfolio_name"]], - investor_name = cfg[["investor_name"]] + portfolio_name = params[["portfolio_name"]], + investor_name = params[["investor_name"]] ) log_debug("Loading total portfolio results.") @@ -141,8 +141,8 @@ run_pacta_reporting_process <- function( ) total_portfolio <- add_inv_and_port_names_if_needed( data = total_portfolio, - portfolio_name = cfg[["portfolio_name"]], - investor_name = cfg[["investor_name"]] + portfolio_name = params[["portfolio_name"]], + investor_name = params[["investor_name"]] ) log_debug("Loading portfolio equity results.") @@ -152,8 +152,8 @@ run_pacta_reporting_process <- function( ) equity_results_portfolio <- add_inv_and_port_names_if_needed( data = equity_results_portfolio, - portfolio_name = cfg[["portfolio_name"]], - investor_name = cfg[["investor_name"]] + portfolio_name = params[["portfolio_name"]], + investor_name = params[["investor_name"]] ) log_debug("Loading portfolio bonds results.") @@ -163,8 +163,8 @@ run_pacta_reporting_process <- function( ) bonds_results_portfolio <- add_inv_and_port_names_if_needed( data = bonds_results_portfolio, - portfolio_name = cfg[["portfolio_name"]], - investor_name = cfg[["investor_name"]] + portfolio_name = params[["portfolio_name"]], + investor_name = params[["investor_name"]] ) log_debug("Loading company equity results.") @@ -174,8 +174,8 @@ run_pacta_reporting_process <- function( ) equity_results_company <- add_inv_and_port_names_if_needed( data = equity_results_company, - portfolio_name = cfg[["portfolio_name"]], - investor_name = cfg[["investor_name"]] + portfolio_name = params[["portfolio_name"]], + investor_name = params[["investor_name"]] ) log_debug("Loading company bonds results.") @@ -185,8 +185,8 @@ run_pacta_reporting_process <- function( ) bonds_results_company <- add_inv_and_port_names_if_needed( data = bonds_results_company, - portfolio_name = cfg[["portfolio_name"]], - investor_name = cfg[["investor_name"]] + portfolio_name = params[["portfolio_name"]], + investor_name = params[["investor_name"]] ) log_debug("Loading equity map results.") @@ -196,8 +196,8 @@ run_pacta_reporting_process <- function( ) equity_results_map <- add_inv_and_port_names_if_needed( data = equity_results_map, - portfolio_name = cfg[["portfolio_name"]], - investor_name = cfg[["investor_name"]] + portfolio_name = params[["portfolio_name"]], + investor_name = params[["investor_name"]] ) log_debug("Loading bonds map results.") @@ -207,8 +207,8 @@ run_pacta_reporting_process <- function( ) bonds_results_map <- add_inv_and_port_names_if_needed( data = bonds_results_map, - portfolio_name = cfg[["portfolio_name"]], - investor_name = cfg[["investor_name"]] + portfolio_name = params[["portfolio_name"]], + investor_name = params[["investor_name"]] ) analysis_output_manifest <- jsonlite::read_json(file.path(analysis_output_dir, "manifest.json")) From eb2e1b16dfff507c4b4a01fd9d9c7e7cf9ebd512 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 16:27:10 +0200 Subject: [PATCH 107/132] remove last references to cfg in top-level script --- R/run_pacta_reporting_process.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 8d75e0c..4e34ea5 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -246,7 +246,7 @@ run_pacta_reporting_process <- function( # create interactive report ---------------------------------------------------- prepare_interactive_report( - cfg = cfg, + cfg = params, report_output_dir = report_output_dir, audit_file = audit_file, emissions = emissions, @@ -268,7 +268,7 @@ run_pacta_reporting_process <- function( # create executive summary ----------------------------------------------------- prepare_executive_summary( - cfg = cfg, + cfg = params, summary_output_dir = summary_output_dir, total_portfolio = total_portfolio, equity_results_portfolio = equity_results_portfolio, From 7cd13008172af92f8e672c93c7ae58c0b9ac9563 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 17:10:07 +0200 Subject: [PATCH 108/132] factor `cfg` out of ES process --- R/prepare_executive_summary.R | 36 +++++++++++++++++++-------------- R/run_pacta_reporting_process.R | 11 +++++++++- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/R/prepare_executive_summary.R b/R/prepare_executive_summary.R index 53c7b87..9ab596a 100644 --- a/R/prepare_executive_summary.R +++ b/R/prepare_executive_summary.R @@ -1,6 +1,15 @@ prepare_executive_summary <- function( - cfg, summary_output_dir, + survey_dir, + real_estate_dir, + score_card_dir, + project_code, + language_select, + peer_group, + investor_name, + portfolio_name, + start_year, + currency_exchange_value, total_portfolio, equity_results_portfolio, bonds_results_portfolio, @@ -15,16 +24,13 @@ prepare_executive_summary <- function( ) { log_debug("Preparing to create executive summary.") - survey_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "survey")) - real_estate_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "real_estate")) - score_card_dir <- fs::path_abs(file.path(cfg$user_results_path, cfg$project_code, "score_card")) es_dir <- file.path(summary_output_dir, "executive_summary") if (!dir.exists(es_dir)) { dir.create(es_dir, showWarnings = FALSE, recursive = TRUE) } log_trace("Defining executive summary template paths.") - exec_summary_template_name <- paste0(cfg$project_code, "_", tolower(cfg$language_select), "_exec_summary") + exec_summary_template_name <- paste0(project_code, "_", tolower(language_select), "_exec_summary") exec_summary_builtin_template_path <- system.file( "extdata", exec_summary_template_name, package = "pacta.executive.summary" @@ -34,16 +40,16 @@ prepare_executive_summary <- function( if ( dir.exists(exec_summary_template_path) && ( - cfg$peer_group %in% c("assetmanager", "bank", "insurance", "pensionfund") + peer_group %in% c("assetmanager", "bank", "insurance", "pensionfund") ) ) { log_debug("Preparing data for executive summary.") data_aggregated_filtered <- pacta.executive.summary::prep_data_executive_summary( - investor_name = cfg$investor_name, - portfolio_name = cfg$portfolio_name, - peer_group = cfg$peer_group, - start_year = cfg$start_year, + investor_name = investor_name, + portfolio_name = portfolio_name, + peer_group = peer_group, + start_year = start_year, scenario_source = "GECO2021", scenario_selected = "1.5C-Unif", scenario_geography = "Global", @@ -81,7 +87,7 @@ prepare_executive_summary <- function( log_info("Creating executive summary.") pacta.executive.summary::render_executive_summary( data = data_aggregated_filtered, - language = cfg$language_select, + language = language_select, output_dir = es_dir, exec_summary_dir = exec_summary_template_path, survey_dir = survey_dir, @@ -89,12 +95,12 @@ prepare_executive_summary <- function( real_estate_flag = real_estate_flag, score_card_dir = score_card_dir, file_name = "template.Rmd", - investor_name = cfg$investor_name, - portfolio_name = cfg$portfolio_name, - peer_group = cfg$peer_group, + investor_name = investor_name, + portfolio_name = portfolio_name, + peer_group = peer_group, total_portfolio = total_portfolio, scenario_selected = "1.5C-Unif", - currency_exchange_value = cfg$currency_exchange_value, + currency_exchange_value = currency_exchange_value, log_dir = summary_output_dir ) } else { diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 4e34ea5..f664591 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -268,8 +268,17 @@ run_pacta_reporting_process <- function( # create executive summary ----------------------------------------------------- prepare_executive_summary( - cfg = params, summary_output_dir = summary_output_dir, + survey_dir = params[["survey_dir"]], + real_estate_dir = params[["real_estate_dir"]], + score_card_dir = params[["score_card_dir"]], + project_code = params[["project_code"]], + language_select = params[["language_select"]], + peer_group = params[["peer_group"]], + investor_name = params[["investor_name"]], + portfolio_name = params[["portfolio_name"]], + start_year = params[["start_year"]], + currency_exchange_value = params[["currency_exchange_value"]], total_portfolio = total_portfolio, equity_results_portfolio = equity_results_portfolio, bonds_results_portfolio = bonds_results_portfolio, From 37696efcf64ac160805ea355b10bc94420cd0d94 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 17:11:16 +0200 Subject: [PATCH 109/132] Fix bad quotes --- R/run_pacta_reporting_process.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index f664591..45e7888 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -93,7 +93,7 @@ run_pacta_reporting_process <- function( log_warn("Overriding language selection to \"EN\" for \"GENERAL\".") params[["language_select"]] <- "EN" } else { - log_trace("Using language selection: \"{params[["language_select"]]}\".") + log_trace("Using language selection: \"{params[['language_select']]}\".") } From daaa6134776fa65b069ba6baa84a147c22ad0576 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 17:59:05 +0200 Subject: [PATCH 110/132] Factor `cfg` out of interactive report process --- R/prepare_interactive_report.R | 63 ++++++++++++++++++++------------- R/run_pacta_reporting_process.R | 19 +++++++++- 2 files changed, 57 insertions(+), 25 deletions(-) diff --git a/R/prepare_interactive_report.R b/R/prepare_interactive_report.R index b29ba1b..c2fe299 100644 --- a/R/prepare_interactive_report.R +++ b/R/prepare_interactive_report.R @@ -1,6 +1,23 @@ prepare_interactive_report <- function( - cfg, report_output_dir, + project_report_name, + survey_dir, + real_estate_dir, + language_select, + peer_group, + investor_name, + portfolio_name, + start_year, + currency_exchange_value, + select_scenario, + scenario_other, + portfolio_allocation_method, + scenario_geography, + sector_list, + green_techs, + tech_roadmap_sectors, + pacta_sectors_not_analysed, + display_currency, audit_file, emissions, portfolio_overview, @@ -46,39 +63,37 @@ prepare_interactive_report <- function( # combine config files to send to create_interactive_report() log_trace("Defining configs and manifest.") - configs <- - list( - portfolio_config = cfg, - analysis_output_manifest = analysis_output_manifest - ) + configs <- list( + analysis_output_manifest = analysis_output_manifest + ) # workaround a bug in {config} v0.3.2 that only adds "config" class to objects it creates class(configs$portfolio_config) <- c(class(configs$portfolio_config), "list") log_trace("Defining interactive report template paths.") template_path <- system.file("templates", package = "pacta.portfolio.report") - template_dir_name <- paste(tolower(cfg$project_report_name), tolower(cfg$language_select), "template", sep = "_") + template_dir_name <- paste(tolower(project_report_name), tolower(language_select), "template", sep = "_") template_dir <- file.path(template_path, template_dir_name) log_info("Creating interactive report.") pacta.portfolio.report::create_interactive_report( template_dir = template_dir, output_dir = report_output_dir, - survey_dir = cfg$survey_dir, - real_estate_dir = cfg$real_estate_dir, - language_select = cfg$language_select, - investor_name = cfg$investor_name, - portfolio_name = cfg$portfolio_name, - peer_group = cfg$peer_group, - start_year = cfg$start_year, - select_scenario = cfg$select_scenario, - select_scenario_other = cfg$scenario_other, - portfolio_allocation_method = cfg$portfolio_allocation_method, - scenario_geography = cfg$scenario_geography, - pacta_sectors = cfg$sector_list, - green_techs = cfg$green_techs, - tech_roadmap_sectors = cfg$tech_roadmap_sectors, - pacta_sectors_not_analysed = cfg$pacta_sectors_not_analysed, + survey_dir = survey_dir, + real_estate_dir = real_estate_dir, + language_select = language_select, + investor_name = investor_name, + portfolio_name = portfolio_name, + peer_group = peer_group, + start_year = start_year, + select_scenario = select_scenario, + select_scenario_other = scenario_other, + portfolio_allocation_method = portfolio_allocation_method, + scenario_geography = scenario_geography, + pacta_sectors = sector_list, + green_techs = green_techs, + tech_roadmap_sectors = tech_roadmap_sectors, + pacta_sectors_not_analysed = pacta_sectors_not_analysed, audit_file = audit_file, emissions = emissions, portfolio_overview = portfolio_overview, @@ -96,8 +111,8 @@ prepare_interactive_report <- function( peers_bonds_results_user = peers_bonds_results_user, dataframe_translations = dataframe_translations, js_translations = js_translations, - display_currency = cfg$display_currency, - currency_exchange_value = cfg$currency_exchange_value, + display_currency = display_currency, + currency_exchange_value = currency_exchange_value, header_dictionary = header_dictionary, sector_order = sector_order, configs = configs diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 45e7888..196d964 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -246,8 +246,25 @@ run_pacta_reporting_process <- function( # create interactive report ---------------------------------------------------- prepare_interactive_report( - cfg = params, report_output_dir = report_output_dir, + project_report_name = params[["project_report_name"]], + survey_dir = params[["survey_dir"]], + real_estate_dir = params[["real_estate_dir"]], + language_select = params[["language_select"]], + peer_group = params[["peer_group"]], + investor_name = params[["investor_name"]], + portfolio_name = params[["portfolio_name"]], + start_year = params[["start_year"]], + currency_exchange_value = params[["currency_exchange_value"]], + select_scenario = params[["select_scenario"]], + scenario_other = params[["scenario_other"]], + portfolio_allocation_method = params[["portfolio_allocation_method"]], + scenario_geography = params[["scenario_geography"]], + sector_list = params[["sector_list"]], + green_techs = params[["green_techs"]], + tech_roadmap_sectors = params[["tech_roadmap_sectors"]], + pacta_sectors_not_analysed = params[["pacta_sectors_not_analysed"]], + display_currency = params[["display_currency"]], audit_file = audit_file, emissions = emissions, portfolio_overview = portfolio_overview, From 44afcf5840e3d91a8bddf069ce6877fd797bd162 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 18:01:42 +0200 Subject: [PATCH 111/132] update lintr sttings --- .lintr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.lintr b/.lintr index 7632ce7..b9931e6 100644 --- a/.lintr +++ b/.lintr @@ -1,4 +1,6 @@ linters: linters_with_defaults( + commented_code_linter = NULL, + cyclocomp_linter = NULL, object_length_linter(35), line_length_linter(120) ) From 9c0a60f6e3f6120e0685b87276e340487589ebd9 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 18:12:53 +0200 Subject: [PATCH 112/132] Remove unused dependency --- DESCRIPTION | 1 - 1 file changed, 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index b38e968..256fbdc 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -22,7 +22,6 @@ Roxygen: list(markdown = TRUE) RoxygenNote: 7.3.1 Imports: dplyr, - fs, jsonlite, logger, pacta.executive.summary, From e093020f74db165962d27d77ff80c44c46f899f9 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 18:16:19 +0200 Subject: [PATCH 113/132] Add sysdeps --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index d94a340..c697b06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,8 +14,12 @@ USER root RUN apt-get update \ && DEBIAN_FRONTEND="noninteractive" \ apt-get install -y --no-install-recommends \ + git=1:2.34.* \ + libcurl4-openssl-dev=7.81.* \ + libgit2-dev=1.1.* \ libicu-dev=70.* \ libpng-dev=1.6.* \ + libssl-dev=3.0.* \ libxt6=1:1.2.* \ pandoc=2.9.* \ && chmod -R a+rwX /root \ From 3071094ebe09819a25d7decb08c4ffb767b844bb Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 18:25:01 +0200 Subject: [PATCH 114/132] Allow test script to read PARAMs --- .github/workflows/test.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 99f4cda..9aefbbe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -117,9 +117,8 @@ jobs: --mount type=bind,readonly,source=${WORKSPACE}/${ANALYSIS_OUTPUT_DIR},target=/mnt/analysis_output_dir \ --mount type=bind,source=${WORKSPACE}/${REPORT_OUTPUT_DIR},target=/mnt/report_output_dir \ --mount type=bind,source=${WORKSPACE}/${SUMMARY_OUTPUT_DIR},target=/mnt/summary_output_dir \ - $FULL_IMAGE_NAME - # $FULL_IMAGE_NAME \ - # $PARAMETERS + $FULL_IMAGE_NAME \ + $PARAMETERS - name: List outputs run: | From 213a10a9990e61ffb802c7c4109f0971f1d92fbf Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 18:31:41 +0200 Subject: [PATCH 115/132] Move file --- .../default_config.json => inst/extdata/parameters/default.json | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename input_dir/default_config.json => inst/extdata/parameters/default.json (100%) diff --git a/input_dir/default_config.json b/inst/extdata/parameters/default.json similarity index 100% rename from input_dir/default_config.json rename to inst/extdata/parameters/default.json From d32cfeb1d27ebfd4e5520a88a1427d2f2d5c4862 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 18:38:49 +0200 Subject: [PATCH 116/132] simplify default params --- inst/extdata/parameters/default.json | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/inst/extdata/parameters/default.json b/inst/extdata/parameters/default.json index bc412ec..547e765 100644 --- a/inst/extdata/parameters/default.json +++ b/inst/extdata/parameters/default.json @@ -1,25 +1,6 @@ { - "data_dir": "/mnt/pacta-data/", - "portfolio_path": "/mnt/input_dir/default_portfolio.csv", - "output_dir": "/mnt/output_dir", - "report_dir": "/mnt/report_dir", "start_year": 2023, - "equity_market_list": [ - "GlobalMarket", - "DevelopedMarket", - "EmergingMarket" - ], - "scenario_sources_list": [ - "GECO2023", - "ISF2023", - "WEO2023" - ], - "scenario_geographies_list": [ - "Global", - "GlobalAggregate", - "NonOECD", - "OECD" - ], + "scenario_geography": "Global", "sector_list": [ "Power", "Automotive", @@ -29,20 +10,17 @@ "Aviation", "Cement" ], - "has_map": false, "project_code": "GENERAL", "portfolio_name": "Default Portfolio", "investor_name": "Default Investor", "peer_group": "bank", "language_select": "EN", - "user_id": "4", "project_report_name": "general", "display_currency": "USD", "currency_exchange_value": 1, "select_scenario": "WEO2022_NZE_2050", "scenario_other": "GECO2022_1.5C", "portfolio_allocation_method": "portfolio_weight", - "scenario_geography": "Global", "tech_roadmap_sectors": [ "Power", "Automotive", From af887c5f971d7246d621c85c26626b32aaf285b1 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 19:07:27 +0200 Subject: [PATCH 117/132] Move params to config --- .github/workflows/test.yml | 7 +++++++ R/run_pacta_reporting_process.R | 4 +++- docker-compose.yml | 5 +++-- inst/extdata/parameters/default.json | 5 +---- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9aefbbe..dd91827 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -101,11 +101,18 @@ jobs: SUMMARY_OUTPUT_DIR: summary_output_dir WORKSPACE: ${{ github.workspace }} PARAMETERS: ${{ steps.prepare.outputs.parameters }} + REAL_ESTATE_DIR: "/mnt/real_estate_dir" + SURVEY_DIR: "/mnt/survey_dir" run: | mkdir -p "$REPORT_OUTPUT_DIR" chmod -R 777 "$REPORT_OUTPUT_DIR" mkdir -p "$SUMMARY_OUTPUT_DIR" chmod -R 777 "$SUMMARY_OUTPUT_DIR" + mkdir -p "$REAL_ESTATE_DIR" + chmod -R 777 "$REAL_ESTATE_DIR" + mkdir -p "$SURVEY_DIR" + chmod -R 777 "$SURVEY_DIR" + docker run \ --network none \ --env LOG_LEVEL=TRACE \ diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 196d964..8bf38ca 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -3,7 +3,9 @@ run_pacta_reporting_process <- function( analysis_output_dir = Sys.getenv("ANALYSIS_OUTPUT_DIR"), benchmarks_dir = Sys.getenv("BENCHMARKS_DIR"), report_output_dir = Sys.getenv("REPORT_OUTPUT_DIR"), - summary_output_dir = Sys.getenv("SUMMARY_OUTPUT_DIR") + summary_output_dir = Sys.getenv("SUMMARY_OUTPUT_DIR"), + real_estate_dir = Sys.getenv("REAL_ESTATE_DIR"), + survey_dir = Sys.getenv("SURVEY_DIR") ) { log_debug("Checking configuration.") if (is.null(analysis_output_dir) || analysis_output_dir == "") { diff --git a/docker-compose.yml b/docker-compose.yml index 8dcb590..eacba53 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,14 +6,15 @@ services: # tty: true # entrypoint: ["R", "--args"] # command: '{\"portfolio_files\": \"default_portfolio.csv\", \"inherit\": \"GENERAL_2023Q4\"}' - # entrypoint: "bash" - # command: "/workflow.pacta.report/input_dir/default_config.json" + command: "/workflow.pacta.report/inst/extdata/parameters/default.json" environment: LOG_LEVEL: TRACE ANALYSIS_OUTPUT_DIR: "/mnt/analysis_output_dir" BENCHMARKS_DIR: "/mnt/benchmarks_dir" REPORT_OUTPUT_DIR: "/mnt/report_output_dir" SUMMARY_OUTPUT_DIR: "/mnt/summary_output_dir" + REAL_ESTATE_DIR: "/mnt/real_estate_dir" + SURVEY_DIR: "/mnt/survey_dir" volumes: - type: bind source: ${benchmarks_dir:-./benchmarks_dir} diff --git a/inst/extdata/parameters/default.json b/inst/extdata/parameters/default.json index 547e765..9cb60aa 100644 --- a/inst/extdata/parameters/default.json +++ b/inst/extdata/parameters/default.json @@ -40,8 +40,5 @@ "Electric", "FuelCell", "Electric Arc Furnace" - ], - "user_results_path": "input_dir/user_results", - "real_estate_dir": "input_dir/real_estate", - "survey_dir": "input_dir/survey" + ] } From 098109d9d37b90bbda5a308ba9b0a45a616e6ff0 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 19:08:17 +0200 Subject: [PATCH 118/132] Remove unneded class assignment --- R/prepare_interactive_report.R | 3 --- 1 file changed, 3 deletions(-) diff --git a/R/prepare_interactive_report.R b/R/prepare_interactive_report.R index c2fe299..f35cbbe 100644 --- a/R/prepare_interactive_report.R +++ b/R/prepare_interactive_report.R @@ -67,9 +67,6 @@ prepare_interactive_report <- function( analysis_output_manifest = analysis_output_manifest ) - # workaround a bug in {config} v0.3.2 that only adds "config" class to objects it creates - class(configs$portfolio_config) <- c(class(configs$portfolio_config), "list") - log_trace("Defining interactive report template paths.") template_path <- system.file("templates", package = "pacta.portfolio.report") template_dir_name <- paste(tolower(project_report_name), tolower(language_select), "template", sep = "_") From 8d7fba448869aa3778c8578a73a5c3bd5d88c4ad Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 19:21:20 +0200 Subject: [PATCH 119/132] Fix call to real estate dir --- R/run_pacta_reporting_process.R | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 8bf38ca..b18e85d 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -5,7 +5,8 @@ run_pacta_reporting_process <- function( report_output_dir = Sys.getenv("REPORT_OUTPUT_DIR"), summary_output_dir = Sys.getenv("SUMMARY_OUTPUT_DIR"), real_estate_dir = Sys.getenv("REAL_ESTATE_DIR"), - survey_dir = Sys.getenv("SURVEY_DIR") + survey_dir = Sys.getenv("SURVEY_DIR"), + score_card_dir = Sys.getenv("SCORE_CARD_DIR") ) { log_debug("Checking configuration.") if (is.null(analysis_output_dir) || analysis_output_dir == "") { @@ -250,8 +251,8 @@ run_pacta_reporting_process <- function( prepare_interactive_report( report_output_dir = report_output_dir, project_report_name = params[["project_report_name"]], - survey_dir = params[["survey_dir"]], - real_estate_dir = params[["real_estate_dir"]], + survey_dir = survey_dir, + real_estate_dir = real_estate_dir, language_select = params[["language_select"]], peer_group = params[["peer_group"]], investor_name = params[["investor_name"]], @@ -288,9 +289,9 @@ run_pacta_reporting_process <- function( # create executive summary ----------------------------------------------------- prepare_executive_summary( summary_output_dir = summary_output_dir, - survey_dir = params[["survey_dir"]], - real_estate_dir = params[["real_estate_dir"]], - score_card_dir = params[["score_card_dir"]], + survey_dir = survey_dir, + real_estate_dir = real_estate_dir, + score_card_dir = score_card_dir, project_code = params[["project_code"]], language_select = params[["language_select"]], peer_group = params[["peer_group"]], From d632575513eb70c2718a69da44034608a6977ef8 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 19:28:06 +0200 Subject: [PATCH 120/132] Add more bind mounts --- docker-compose.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index eacba53..4bd7877 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -24,6 +24,14 @@ services: source: ${analysis_output_dir:-./analysis_output_dir} target: /mnt/analysis_output_dir read_only: true + - type: bind + source: ${real_estate_dir:-./real_estate_dir} + target: /mnt/real_estate_dir + read_only: true + - type: bind + source: ${survey_dir:-./survey_dir} + target: /mnt/survey_dir + read_only: true - type: bind source: ${report_output_dir:-./report_output_dir} target: /mnt/report_output_dir From 16fb696e401b3538a555e62b72aaf6bdeafae01b Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 19:30:18 +0200 Subject: [PATCH 121/132] Fix bad path in test suite --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dd91827..48e44bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -101,8 +101,8 @@ jobs: SUMMARY_OUTPUT_DIR: summary_output_dir WORKSPACE: ${{ github.workspace }} PARAMETERS: ${{ steps.prepare.outputs.parameters }} - REAL_ESTATE_DIR: "/mnt/real_estate_dir" - SURVEY_DIR: "/mnt/survey_dir" + REAL_ESTATE_DIR: real_estate_dir + SURVEY_DIR: survey_dir run: | mkdir -p "$REPORT_OUTPUT_DIR" chmod -R 777 "$REPORT_OUTPUT_DIR" From d3ecd912da5fef9d9d39c8b2b945b6f2bd130c08 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 20:01:35 +0200 Subject: [PATCH 122/132] Explicitly no validation yet --- R/run_pacta_reporting_process.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index b18e85d..97a4744 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -66,7 +66,8 @@ run_pacta_reporting_process <- function( inheritence_search_paths = system.file( "extdata", "parameters", package = "workflow.pacta.report" - ) #, + ) , + schema_file = NULL # schema_file = system.file( # "extdata", "schema", "portfolioParameters_0-0-1.json", # package = "workflow.pacta.report" From c1c5e736d7df920ca3158d616413bf76e46779aa Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 20:05:23 +0200 Subject: [PATCH 123/132] Add inheritance parameters --- docker-compose.yml | 3 +-- inst/extdata/parameters/GENERAL.json | 5 +++++ inst/extdata/parameters/GENERAL_2022Q4.json | 6 ++++++ inst/extdata/parameters/GENERAL_2023Q4.json | 6 ++++++ inst/extdata/parameters/default.json | 4 ---- 5 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 inst/extdata/parameters/GENERAL.json create mode 100644 inst/extdata/parameters/GENERAL_2022Q4.json create mode 100644 inst/extdata/parameters/GENERAL_2023Q4.json diff --git a/docker-compose.yml b/docker-compose.yml index 4bd7877..5325c70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,8 +5,7 @@ services: # stdin_open: true # tty: true # entrypoint: ["R", "--args"] - # command: '{\"portfolio_files\": \"default_portfolio.csv\", \"inherit\": \"GENERAL_2023Q4\"}' - command: "/workflow.pacta.report/inst/extdata/parameters/default.json" + command: '{\"portfolio_files\": \"default_portfolio.csv\", \"inherit\": \"GENERAL_2023Q4\"}' environment: LOG_LEVEL: TRACE ANALYSIS_OUTPUT_DIR: "/mnt/analysis_output_dir" diff --git a/inst/extdata/parameters/GENERAL.json b/inst/extdata/parameters/GENERAL.json new file mode 100644 index 0000000..8012eb5 --- /dev/null +++ b/inst/extdata/parameters/GENERAL.json @@ -0,0 +1,5 @@ +{ + "project_code": "GENERAL", + "project_report_name": "general", + "inherit": "default" +} diff --git a/inst/extdata/parameters/GENERAL_2022Q4.json b/inst/extdata/parameters/GENERAL_2022Q4.json new file mode 100644 index 0000000..05c64c9 --- /dev/null +++ b/inst/extdata/parameters/GENERAL_2022Q4.json @@ -0,0 +1,6 @@ +{ + "select_scenario": "WEO2022_NZE_2050", + "scenario_other": "GECO2022_1.5C", + "inherit": "GENERAL" +} + diff --git a/inst/extdata/parameters/GENERAL_2023Q4.json b/inst/extdata/parameters/GENERAL_2023Q4.json new file mode 100644 index 0000000..05c64c9 --- /dev/null +++ b/inst/extdata/parameters/GENERAL_2023Q4.json @@ -0,0 +1,6 @@ +{ + "select_scenario": "WEO2022_NZE_2050", + "scenario_other": "GECO2022_1.5C", + "inherit": "GENERAL" +} + diff --git a/inst/extdata/parameters/default.json b/inst/extdata/parameters/default.json index 9cb60aa..e9c7436 100644 --- a/inst/extdata/parameters/default.json +++ b/inst/extdata/parameters/default.json @@ -10,16 +10,12 @@ "Aviation", "Cement" ], - "project_code": "GENERAL", "portfolio_name": "Default Portfolio", "investor_name": "Default Investor", "peer_group": "bank", "language_select": "EN", - "project_report_name": "general", "display_currency": "USD", "currency_exchange_value": 1, - "select_scenario": "WEO2022_NZE_2050", - "scenario_other": "GECO2022_1.5C", "portfolio_allocation_method": "portfolio_weight", "tech_roadmap_sectors": [ "Power", From 928afe28aa7b5915ea03df715e7f1e6d0ff2f863 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Fri, 31 May 2024 20:10:16 +0200 Subject: [PATCH 124/132] Disable test --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3011bd0..9274fd8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: config-name: - default_2022Q4 - default_2023Q4 - - full_params_2023Q4 + # - full_params_2023Q4 with: full-image-name: ${{ needs.docker.outputs.full-image-name }} config-name: ${{ matrix.config-name }} From 9bcdc6ade8146cf1eabffbbb41c5cff3df920682 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 3 Jun 2024 11:56:19 +0200 Subject: [PATCH 125/132] Fix output paths in test --- .github/workflows/test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 48e44bb..821555a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -148,13 +148,13 @@ jobs: inlineScript: | unique_directory="$RESULTS_URL/$GITHUB_REF_NAME/$GITHUB_RUN_NUMBER/$GITHUB_RUN_ATTEMPT/$CONFIG_NAME" az storage copy \ - --source "$REPORT_OUTPUT_DIR" \ + --source "$REPORT_OUTPUT_DIR"/* \ --destination "$unique_directory" \ --recursive echo "report-url=${unique_directory}/report/index.html" >> "$GITHUB_OUTPUT" az storage copy \ - --source "$SUMMARY_OUTPUT_DIR" \ + --source "$SUMMARY_OUTPUT_DIR"/* \ --destination "$unique_directory" \ --recursive summary_files="$(ls $SUMMARY_OUTPUT_DIR/executive_summary/*.pdf)" From ed70eacabf4a811286a8d948ddb8d131f19db3ff Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 3 Jun 2024 14:38:56 +0200 Subject: [PATCH 126/132] Use main branch for dependency https://github.com/RMI-PACTA/pacta.portfolio.report/pull/78 is merged --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 256fbdc..89824c6 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -31,6 +31,6 @@ Imports: readr Remotes: RMI-PACTA/pacta.executive.summary, - RMI-PACTA/pacta.portfolio.report@issues-found-in-workflow.pacta.report, + RMI-PACTA/pacta.portfolio.report, RMI-PACTA/pacta.portfolio.utils, RMI-PACTA/pacta.workflow.utils From 0b711fa492c5ca81fabd17e12561c28a2e918fb9 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 3 Jun 2024 15:32:19 +0200 Subject: [PATCH 127/132] Move `start_year` param to year-specific files --- inst/extdata/parameters/GENERAL_2022Q4.json | 1 + inst/extdata/parameters/GENERAL_2023Q4.json | 1 + inst/extdata/parameters/default.json | 1 - 3 files changed, 2 insertions(+), 1 deletion(-) diff --git a/inst/extdata/parameters/GENERAL_2022Q4.json b/inst/extdata/parameters/GENERAL_2022Q4.json index 05c64c9..cb859b0 100644 --- a/inst/extdata/parameters/GENERAL_2022Q4.json +++ b/inst/extdata/parameters/GENERAL_2022Q4.json @@ -1,6 +1,7 @@ { "select_scenario": "WEO2022_NZE_2050", "scenario_other": "GECO2022_1.5C", + "start_year": 2022, "inherit": "GENERAL" } diff --git a/inst/extdata/parameters/GENERAL_2023Q4.json b/inst/extdata/parameters/GENERAL_2023Q4.json index 05c64c9..9afadd7 100644 --- a/inst/extdata/parameters/GENERAL_2023Q4.json +++ b/inst/extdata/parameters/GENERAL_2023Q4.json @@ -1,6 +1,7 @@ { "select_scenario": "WEO2022_NZE_2050", "scenario_other": "GECO2022_1.5C", + "start_year": 2023, "inherit": "GENERAL" } diff --git a/inst/extdata/parameters/default.json b/inst/extdata/parameters/default.json index e9c7436..2ccad1b 100644 --- a/inst/extdata/parameters/default.json +++ b/inst/extdata/parameters/default.json @@ -1,5 +1,4 @@ { - "start_year": 2023, "scenario_geography": "Global", "sector_list": [ "Power", From 6f73ddb02a51cdc9da90f57ada74e3a860dde4eb Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Mon, 3 Jun 2024 17:06:12 +0200 Subject: [PATCH 128/132] Quic update for readme --- README.md | 40 +--------------------------------------- 1 file changed, 1 insertion(+), 39 deletions(-) diff --git a/README.md b/README.md index 82bca40..45c297e 100644 --- a/README.md +++ b/README.md @@ -11,42 +11,4 @@ packages that it depends on. Running PACTA also requires pacta-data, which needs to be mounted into the container at run-time. -# Using Docker images pushed to GHCR automatically by GH Actions - -``` {.bash} - -tag_name=main -image_name=ghcr.io/rmi-pacta/workflow.pacta.report:$tag_name -data_dir=~/Downloads/2022Q4_transition_monitor_inputs_2023-07-11/ -input_dir=./input_dir -output_dir=./output_dir - -# Build -docker build . -t $image_name - -# Run -docker run --rm \ - --network none \ - --platform linux/amd64 \ - --env LOG_LEVEL=DEBUG \ - --mount type=bind,readonly,source=${data_dir},target=/pacta-data \ - --mount type=bind,source=${output_dir},target=/output_dir \ - --mount type=bind,source=${input_dir},target=/input_dir \ - $image_name - -``` - -```sh - -# Run R in container -docker run -it --rm \ - --network none \ - --platform linux/amd64 \ - --env LOG_LEVEL=TRACE \ - --mount type=bind,readonly,source=${data_dir},target=/pacta-data \ - --mount type=bind,source=${output_dir},target=/output_dir \ - --mount type=bind,source=${input_dir},target=/input_dir \ - --entrypoint R - $image_name - -``` +See workflow.pacta for examples of a similar docker container and invokation From ef3c845c8be5f18f6bd4fc9d7ab60ec6f3d80291 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 4 Jun 2024 17:10:42 +0200 Subject: [PATCH 129/132] Apply suggestions from code review Co-authored-by: CJ Yetman --- Dockerfile | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c697b06..1a1fc0c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update \ # set frozen CRAN repo and RProfile.site # This block makes use of the builtin ARG $TARGETPLATFORM (See: # https://www.docker.com/blog/faster-multi-platform-builds-dockerfile-cross-compilation-guide/ -# ) to pick the correct CRAN-like repo, which will let us target binaries fo +# ) to pick the correct CRAN-like repo, which will let us target binaries for # supported platforms ARG TARGETPLATFORM RUN PACKAGE_PIN_DATE="2024-04-05" && \ diff --git a/README.md b/README.md index 45c297e..cfb35aa 100644 --- a/README.md +++ b/README.md @@ -11,4 +11,4 @@ packages that it depends on. Running PACTA also requires pacta-data, which needs to be mounted into the container at run-time. -See workflow.pacta for examples of a similar docker container and invokation +See workflow.pacta for examples of a similar docker container and invocation From 2ee35470c6cc364d0138e33d05eb46e8353b4f7a Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 4 Jun 2024 17:17:58 +0200 Subject: [PATCH 130/132] reindent block --- .github/workflows/test.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 821555a..6490b27 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -114,18 +114,18 @@ jobs: chmod -R 777 "$SURVEY_DIR" docker run \ - --network none \ - --env LOG_LEVEL=TRACE \ - --env ANALYSIS_OUTPUT_DIR="/mnt/analysis_output_dir" \ - --env BENCHMARKS_DIR="/mnt/benchmarks_dir" \ - --env REPORT_OUTPUT_DIR="/mnt/report_output_dir" \ - --env SUMMARY_OUTPUT_DIR="/mnt/summary_output_dir" \ - --mount type=bind,readonly,source=${WORKSPACE}/${BENCHMARKS_DIR},target=/mnt/benchmarks_dir \ - --mount type=bind,readonly,source=${WORKSPACE}/${ANALYSIS_OUTPUT_DIR},target=/mnt/analysis_output_dir \ - --mount type=bind,source=${WORKSPACE}/${REPORT_OUTPUT_DIR},target=/mnt/report_output_dir \ - --mount type=bind,source=${WORKSPACE}/${SUMMARY_OUTPUT_DIR},target=/mnt/summary_output_dir \ - $FULL_IMAGE_NAME \ - $PARAMETERS + --network none \ + --env LOG_LEVEL=TRACE \ + --env ANALYSIS_OUTPUT_DIR="/mnt/analysis_output_dir" \ + --env BENCHMARKS_DIR="/mnt/benchmarks_dir" \ + --env REPORT_OUTPUT_DIR="/mnt/report_output_dir" \ + --env SUMMARY_OUTPUT_DIR="/mnt/summary_output_dir" \ + --mount type=bind,readonly,source=${WORKSPACE}/${BENCHMARKS_DIR},target=/mnt/benchmarks_dir \ + --mount type=bind,readonly,source=${WORKSPACE}/${ANALYSIS_OUTPUT_DIR},target=/mnt/analysis_output_dir \ + --mount type=bind,source=${WORKSPACE}/${REPORT_OUTPUT_DIR},target=/mnt/report_output_dir \ + --mount type=bind,source=${WORKSPACE}/${SUMMARY_OUTPUT_DIR},target=/mnt/summary_output_dir \ + $FULL_IMAGE_NAME \ + $PARAMETERS - name: List outputs run: | From 15f0ba495a5dc5af94915fd95a989879ebfc8d86 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 4 Jun 2024 17:22:32 +0200 Subject: [PATCH 131/132] Fix docker label --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1a1fc0c..8fc65d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ LABEL org.opencontainers.image.description="Docker image to create PACTA reports LABEL org.opencontainers.image.licenses=MIT LABEL org.opencontainers.image.title="workflow.pacta.report" LABEL org.opencontainers.image.vendor="RMI" -LABEL org.opencontainers.image.base.name="ghcr.io/rmi-pacta/workflow.pacta:main" +LABEL org.opencontainers.image.base.name="docker.io/rocker/r-ver:4.3.1" LABEL org.opencontainers.image.authors="Alex Axthelm" # install system dependencies From e778ffcb4eacbd13cf55d04ffff7538f4f03d578 Mon Sep 17 00:00:00 2001 From: Alex Axthelm Date: Tue, 4 Jun 2024 17:29:06 +0200 Subject: [PATCH 132/132] Update error string --- R/run_pacta_reporting_process.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/run_pacta_reporting_process.R b/R/run_pacta_reporting_process.R index 97a4744..a599b45 100644 --- a/R/run_pacta_reporting_process.R +++ b/R/run_pacta_reporting_process.R @@ -84,7 +84,7 @@ run_pacta_reporting_process <- function( pacta.portfolio.utils::quit_if_no_pacta_relevant_data(total_portfolio) } else { log_warn("file \"{total_portfolio_path}\" does not exist.") - warning("This is weird... the `total_portfolio.rds` file does not exist in the `30_Processed_inputs` directory.") + warning("Cannot find total_portfolio.rds file. Exiting.") }