Feat/34 enable parameter parsing

.Rbuildignore

@@ -7,3 +7,4 @@
 ^\.Rproj\.user$
 ^docker-compose.yml$
 ^dashboard_output_dir
+^pacta-data/

.dockerignore

@@ -11,3 +11,5 @@ benchmarks_dir/**/*
 dashboard_data_dir
 dashboard_data_dir/**/*
 docker-compose.yml
+pacta-data/**/*
+^tests/**/*

.github/workflows/docker.yml

@@ -21,10 +21,27 @@ jobs:
     with:
       do-check-r-sysdeps: false
 
+  test:
+    name: Test
+    uses: ./.github/workflows/test.yml
+    needs: [docker]
+    secrets: inherit
+    strategy:
+      fail-fast: false
+      matrix:
+        config-name:
+          - default_2022Q4
+          - default_2023Q4
+          - full_params_2022Q4
+          - full_params_2023Q4
+    with:
+      full-image-name: ${{ needs.docker.outputs.full-image-name }}
+      config-name: ${{ matrix.config-name }}
+
   gh-pages:
     name: gh-pages
     uses: ./.github/workflows/gh-pages.yml
-    needs: [docker]
+    needs: [docker, test]
     secrets: inherit
     with:
       full-image-name: ${{ needs.docker.outputs.full-image-name }}

.github/workflows/gh-pages.yml

@@ -1,3 +1,4 @@
+---
 name: Deploy to GitHub Pages
 
 on:
@@ -21,68 +22,35 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended
-      - name: Azure Login
-        uses: azure/login@v2
+      - uses: actions/download-artifact@v4
         with:
-          client-id: ${{ secrets.AZURE_CLIENT_ID }}
-          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
-          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          pattern: dashboard-build-*
+          merge-multiple: false
+          path: extracted_builds
 
-      - name: Download Analysis Outputs
-        id: download-analysis-outputs
-        uses: RMI-PACTA/actions/actions/azure/blob-copy@main
-        with:
-          source: https://pactadatadev.blob.core.windows.net/ghactions-workflow-pacta-results/main/latest/full_params_2023Q4/analysis_output_dir
-          destination: tmp_analysis_output_dir
-
-      - name: Download Benchmarks
-        id: download-benchmarks
-        uses: RMI-PACTA/actions/actions/azure/blob-copy@main
-        with:
-          source: https://pactadatadev.blob.core.windows.net/benchmarks-webapp/2023Q4/2023Q4_20240529T002355Z
-          destination: tmp_benchmarks_dir
-
-      - name: move output and analysis files to correct directory
-        run: |
-          mv tmp_analysis_output_dir/main/latest/full_params_2023Q4/analysis_output_dir analysis_output_dir
-          ls analysis_output_dir
-          mv tmp_benchmarks_dir/2023Q4/2023Q4_20240529T002355Z benchmarks_dir
-          ls benchmarks_dir
-
-      - name: Create Output Directories
-        run: |
-          mkdir -p dashboard_output_dir/data
-
-      - name: Run Docker Image
-        env:
-          FULL_IMAGE_NAME: ${{ inputs.full-image-name }}
-          WORKSPACE: ${{ github.workspace }}
-          PARAMETERS: ""
-          ANALYSIS_OUTPUT_DIR: "analysis_output_dir"
-          BENCHMARKS_DIR: "benchmarks_dir"
-          DASHBOARD_DATA_DIR: "dashboard_output_dir/data"
-          DASHBOARD_OUTPUT_DIR: "dashboard_output_dir"
+      - name: Generate PR landing page
+        working-directory: extracted_builds
+        shell: bash
         run: |
 
-          docker run \
-          --network none \
-          --user $(id -u):$(id -g) \
-          --env LOG_LEVEL=TRACE \
-          --env ANALYSIS_OUTPUT_DIR="/mnt/analysis_output_dir" \
-          --env BENCHMARKS_DIR="/mnt/benchmarks_dir" \
-          --env DASHBOARD_DATA_DIR="/mnt/dashboard_output_dir/data" \
-          --env DASHBOARD_OUTPUT_DIR="/mnt/dashboard_output_dir" \
-          --mount type=bind,readonly,source=${WORKSPACE}/${BENCHMARKS_DIR},target=/mnt/benchmarks_dir \
-          --mount type=bind,source=${WORKSPACE}/${ANALYSIS_OUTPUT_DIR},target=/mnt/analysis_output_dir \
-          --mount type=bind,source=${WORKSPACE}/${DASHBOARD_OUTPUT_DIR},target=/mnt/dashboard_output_dir \
-          --mount type=bind,source=${WORKSPACE}/${DASHBOARD_DATA_DIR},target=/mnt/dashboard_output_dir/data \
-          $FULL_IMAGE_NAME \
-          "$PARAMETERS"
+          echo "<!DOCTYPE html><html><body>" > index.html
+          for target_page in $(find */index.html)
+          do
+            echo "$target_page"
+            link_text="$( \
+              echo "$target_page" | \
+              sed 's/^dashboard-build-//' | \
+              sed 's/\/index\.html//' \
+            )"
+            echo "$link_text"
+            echo "<a href=\"$target_page\">$link_text</a><br>" >> index.html
+          done
+          echo "</body></html>" >> index.html
+          cat index.html
 
-      - name: chmod & List outputs
+      - name: Show files
         run: |
-          ls -lR dashboard_output_dir
+          ls -laR extracted_builds
 
       - name: Add nojekyll file
         if: github.event.action != 'closed'
@@ -103,7 +71,7 @@ jobs:
         if: ${{ github.event_name == 'pull_request' }}
         uses: rossjrw/pr-preview-action@v1
         with:
-          source-dir: dashboard_output_dir
+          source-dir: extracted_builds
           preview-branch: gh-pages
           umbrella-dir: pr-preview
           action: auto

.github/workflows/test.yml

@@ -0,0 +1,153 @@
+---
+name: Test docker image
+
+on:
+  workflow_call:
+    inputs:
+      full-image-name:
+        required: true
+        type: string
+      config-name:
+        required: true
+        type: string
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    steps:
+
+      - name: Checkout workflow
+        uses: actions/checkout@v4
+
+      - name: Prepare environment
+        id: prepare
+        env:
+          CONFIG_NAME: ${{ inputs.config-name }}
+          GITHUB_REF_NAME: ${{ github.ref_name}}
+          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
+          GITHUB_RUN_NUMBER: ${{ github.run_number }}
+          WORKSPACE: ${{ github.workspace }}
+        run: |
+          config_file="tests/config/$CONFIG_NAME.json"
+          echo "config_file: $config_file"
+
+          HOLDINGS_DATE="$(jq -rc '.holdingsDate' $config_file)"
+          echo "holdings-date=$HOLDINGS_DATE"
+          echo "holdings-date=$HOLDINGS_DATE" >> "$GITHUB_OUTPUT"
+
+          PACTA_DATA_URL="$(jq -rc '.pactaDataURL' $config_file)"
+          echo "pacta-data-url=$PACTA_DATA_URL"
+          echo "pacta-data-url=$PACTA_DATA_URL" >> "$GITHUB_OUTPUT"
+
+          BENCHMARKS_URL="$(jq -rc '.benchmarksURL' $config_file)"
+          echo "benchmarks-url=$BENCHMARKS_URL"
+          echo "benchmarks-url=$BENCHMARKS_URL" >> "$GITHUB_OUTPUT"
+
+          # includes handling for null/missing keys
+          PARAMETERS="$(jq -rc '.parameters | select( . != null )' $config_file)"
+          echo "parameters=$PARAMETERS"
+          echo "parameters=$PARAMETERS" >> "$GITHUB_OUTPUT"
+
+          TEST_DIR="$WORKSPACE/$GITHUB_REF_NAME/$GITHUB_RUN_NUMBER/$GITHUB_RUN_ATTEMPT/$CONFIG_NAME"
+          mkdir -p $TEST_DIR
+          echo "test-dir=$TEST_DIR"
+          echo "test-dir=$TEST_DIR" >> "$GITHUB_OUTPUT"
+
+          DASHBOARD_OUTPUT_DIR="$TEST_DIR/dashboard_output_dir"
+          mkdir -p "$DASHBOARD_OUTPUT_DIR"
+          chmod -R 777 "$DASHBOARD_OUTPUT_DIR"
+          echo "dashboard-output-dir=$DASHBOARD_OUTPUT_DIR"
+          echo "dashboard-output-dir=$DASHBOARD_OUTPUT_DIR" >> "$GITHUB_OUTPUT"
+
+          DASHBOARD_DATA_DIR="$DASHBOARD_OUTPUT_DIR/data"
+          mkdir -p "$DASHBOARD_DATA_DIR"
+          chmod -R 777 "$DASHBOARD_DATA_DIR"
+          echo "dashboard-data-dir=$DASHBOARD_DATA_DIR"
+          echo "dashboard-data-dir=$DASHBOARD_DATA_DIR" >> "$GITHUB_OUTPUT"
+
+          ANALYSIS_OUTPUT_DIR="$TEST_DIR/analysis_output_dir"
+          mkdir -p "$ANALYSIS_OUTPUT_DIR"
+          chmod -R 777 "$ANALYSIS_OUTPUT_DIR"
+          echo "analysis-output-dir=$ANALYSIS_OUTPUT_DIR"
+          echo "analysis-output-dir=$ANALYSIS_OUTPUT_DIR" >> "$GITHUB_OUTPUT"
+
+      # https://github.com/Azure/login?tab=readme-ov-file#login-with-openid-connect-oidc-recommended
+      - name: Azure Login
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Download pacta-data
+        id: download-pacta-data
+        uses: RMI-PACTA/actions/actions/azure/blob-copy@main
+        with:
+          source: ${{ steps.prepare.outputs.pacta-data-url }}
+          destination: tmp_pacta_data_dir
+
+      - name: Download Benchmarks
+        id: download-benchmarks
+        uses: RMI-PACTA/actions/actions/azure/blob-copy@main
+        with:
+          source: ${{ steps.prepare.outputs.benchmarks-url }}
+          destination: tmp_benchmarks_dir
+
+      - name: Show TEST_DIR
+        id: ls-test-dir
+        env:
+          TEST_DIR: ${{ steps.prepare.outputs.test-dir }}
+        run: |
+          mkdir -p pacta-data
+          mv tmp_pacta_data_dir/*/*/* pacta-data
+          mkdir -p benchmarks_dir
+          mv tmp_benchmarks_dir/*/*/* benchmarks_dir
+          ls -laR "$TEST_DIR"
+
+      - name: Run Docker Image
+        env:
+          FULL_IMAGE_NAME: ${{ inputs.full-image-name }}
+          WORKSPACE: ${{ github.workspace }}
+          PARAMETERS: ${{ steps.prepare.outputs.parameters }}
+          ANALYSIS_OUTPUT_DIR: ${{ steps.prepare.outputs.analysis-output-dir }}
+          BENCHMARKS_DIR: benchmarks_dir
+          PACTA_DATA_DIR: pacta-data
+          PORTFOLIO_DIR: tests/portfolios
+          DASHBOARD_DATA_DIR: ${{ steps.prepare.outputs.dashboard-data-dir }}
+          DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }}
+        run: |
+
+          docker run \
+          --network none \
+          --env LOG_LEVEL=TRACE \
+          --env ANALYSIS_OUTPUT_DIR="/mnt/analysis_output_dir" \
+          --env OUTPUT_DIR="/mnt/analysis_output_dir" \
+          --env BENCHMARKS_DIR="/mnt/benchmarks_dir" \
+          --env PACTA_DATA_DIR="/mnt/pacta-data" \
+          --env PORTFOLIO_DIR="/mnt/portfolios" \
+          --env DASHBOARD_OUTPUT_DIR="/mnt/dashboard_output_dir" \
+          --env DASHBOARD_DATA_DIR="/mnt/dashboard_output_dir/data" \
+          --mount type=bind,readonly,source=${WORKSPACE}/${BENCHMARKS_DIR},target=/mnt/benchmarks_dir \
+          --mount type=bind,readonly,source=${WORKSPACE}/${PACTA_DATA_DIR},target=/mnt/pacta-data \
+          --mount type=bind,readonly,source=${WORKSPACE}/${PORTFOLIO_DIR},target=/mnt/portfolios \
+          --mount type=bind,source=${ANALYSIS_OUTPUT_DIR},target=/mnt/analysis_output_dir \
+          --mount type=bind,source=${DASHBOARD_OUTPUT_DIR},target=/mnt/dashboard_output_dir \
+          --mount type=bind,source=${DASHBOARD_DATA_DIR},target=/mnt/dashboard_output_dir/data \
+          $FULL_IMAGE_NAME \
+          "$PARAMETERS"
+
+      - name: List outputs
+        env:
+          DASHBOARD_OUTPUT_DIR: ${{ steps.prepare.outputs.dashboard-output-dir }}
+        run: |
+          ls -lR $DASHBOARD_OUTPUT_DIR
+
+      - name: Upload built dashboard as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dashboard-build-${{ inputs.config-name }}
+          path: ${{ steps.prepare.outputs.dashboard-output-dir }}
+          if-no-files-found: error

.gitignore

@@ -5,3 +5,4 @@ analysis_output_dir/**/*
 benchmarks_dir/**/*
 dashboard_output_dir/**/*
 !dashboard_output_dir/data/.gitkeep
+pacta-data/**/*

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: workflow.pacta.dashboard
 Title: Run PACTA dashboard JSON generation
-Version: 0.0.0.9006
+Version: 0.0.0.9008
 Authors@R:
     c(person(given = "Alex",
              family = "Axthelm",
@@ -29,11 +29,16 @@ Imports:
     dplyr,
     glue,
     jsonlite,
+    logger,
     magrittr,
     pacta.portfolio.utils,
+    pacta.workflow.utils,
     readr,
     rlang,
     stringr,
-    tidyr
+    tidyr,
+    workflow.pacta
 Remotes:
-    RMI-PACTA/pacta.portfolio.utils
+    RMI-PACTA/pacta.portfolio.utils,
+    RMI-PACTA/pacta.workflow.utils,
+    RMI-PACTA/workflow.pacta

NAMESPACE

@@ -1,5 +1,7 @@
 # Generated by roxygen2: do not edit by hand
 
+export(prepare_schema_files)
+export(run_dashboard_workflow)
 importFrom(dplyr,across)
 importFrom(dplyr,all_of)
 importFrom(dplyr,arrange)
@@ -30,6 +32,11 @@ importFrom(dplyr,summarise)
 importFrom(dplyr,tibble)
 importFrom(dplyr,transmute)
 importFrom(dplyr,ungroup)
+importFrom(logger,log_debug)
+importFrom(logger,log_error)
+importFrom(logger,log_info)
+importFrom(logger,log_trace)
+importFrom(logger,log_warn)
 importFrom(magrittr,"%>%")
 importFrom(rlang,":=")
 importFrom(rlang,.data)