From f0ec1bb2e7ffb84718fa6f86cb7fa1a98892ce12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daphn=C3=A9=20Grasselly?=
 <56442075+dgrassellyb@users.noreply.github.com>
Date: Mon, 18 Sep 2023 10:04:13 +0200
Subject: [PATCH 1/2] 618 templates action (#145)

* add templates run (draft)

* add some updates

* fix variables

* fix

* fix

* fix

* fix env

* tmp fix for pharmaversesdtm

* debug step

* fix templates folder

* fix path

* fix data path

* add csv format

* fix output dir artifacts

* fix diff when folder unversionned

* fix typo

* typo

* add safe dir

* fix git diff

* fixes

* fix

* fix grep errors

* fixes

* fix

* change artifact path

* zip templates data

* fix zip

* fix zip issues

* fix typo

* typo

* fix paths again

* retry

* fix

* fixes

* fix typo

* fix wrong repo path

* add checkout token

* add secrets: inherit

* rollback

* debug traces

* debug

* retry

* fix json parsing error

* fix repo name

* fix typo

* add suffix names

* fix json access

* fix js var

* fix PR wrong repo

* move data to inst/extdata

* replace rds with rda format

* rollback rds format

* rollback rda

* fix load rda

* update source branch name

* fix condition

* fix git diff

* fix diff

* move rda into data folder

* add data dir

* exclude files and add auto-doc

* fix typo

* typo

* test exclude dataset

* fix zip and change defaults inputs

* fix doc

* improve auto-doc and detect also doc changes

* fix missing artifact

* fix

* add roxygen step

* fix doc

* fix

* fix doc examples

* fix

* add exclude input for spellchecks

* fix examples on doc

* add push-templates-data  input

* change condition

* dynamically rename rda vars

* fix

* debug traces

* fix saving rda format

* fix

* add reviewers

* add spellchecks updates

* clean

* clean

* templates - update vaccine PR reviewers

---------

Co-authored-by: pharmaverse-bot <113703390+pharmaverse-bot@users.noreply.github.com>
---
 .github/workflows/check-templates.yml | 280 ++++++++++++++++++++++++++
 .github/workflows/spellcheck.yml      |   6 +
 2 files changed, 286 insertions(+)

diff --git a/.github/workflows/check-templates.yml b/.github/workflows/check-templates.yml
index be0597c0..d373e019 100644
--- a/.github/workflows/check-templates.yml
+++ b/.github/workflows/check-templates.yml
@@ -7,6 +7,16 @@ on:
         default: '4.1'
         required: false
         type: string
+      push-templates-data:
+        description: 'Push generated templates data to pharmaverseadam repo'
+        default: false
+        required: false
+        type: boolean
+      exclude-templates-data:
+        description: 'List of data generated by templates to exclude (templates script will run, but results will not be push to pharmaverseadam repo) - comma seperated list'
+        default: ''
+        required: false
+        type: string
   push:
     branches:
       - main
@@ -18,6 +28,29 @@ on:
       - devel
       - pre-release
 
+env:
+  # branch name for PR
+  source-branch: update_templates_data
+  repo: pharmaverseadam
+  target-branch: main
+  reviewers: >
+    {
+      "admiralonco": {
+        "reviewers": ["bundfussr", "manciniedoardo"]
+      },
+      "admiralophtha": {
+        "reviewers": ["manciniedoardo"]
+      },
+      "admiral": {
+        "reviewers": ["manciniedoardo", "bms63"]
+      },
+      "admiralvaccine": {
+        "reviewers": ["ahasoplakus", "arjoon-r", "manciniedoardo"]
+      }
+    }
+  # data folder name for templates
+  templates_data_folder: ./data/
+
 name: Check Templates
 
 concurrency:
@@ -164,3 +197,250 @@ jobs:
             }
           }
         shell: Rscript {0}
+
+      - name: Add other data formats and suffixes, exclude data
+        if: inputs.push-templates-data
+        run: |
+          library(readxl)
+          library(zip)
+          loadRData <- function(fileName){
+            load(fileName)
+            get(ls()[ls() != "fileName"])
+          }
+          print(file.path("${{ github.workspace }}", "tmp"))
+          print(file.path(Sys.getenv("GITHUB_WORKSPACE"), "tmp"))
+          folder_path <- file.path(Sys.getenv("GITHUB_WORKSPACE"), "tmp")
+          rda_files <- list.files(path = folder_path, pattern = "\\.rda$")
+
+          ex_files <- unlist(strsplit("${{ inputs.exclude-templates-data }}", ",")) # files to exclude
+
+          for (rda_file in rda_files) {
+            # delete file if part of inputs.exclude-templates-data
+            if (gsub("\\.rda$", "", rda_file) %in% ex_files) {
+              file.remove(file.path(folder_path, rda_file))
+              cat("Deleted:", rda_file, "\n")
+            }
+            else {
+              print(sprintf("converting file %s", rda_file))
+              data <- loadRData(file.path(folder_path, rda_file))
+              suffix <- gsub("pharmaverse/admiral", "", "${{ github.repository }}")
+              if (nchar(suffix) > 0) {
+                rda_file_renamed <- gsub("\\.rda$", sprintf("_%s.rda", suffix), rda_file)
+                csv_file <- gsub("\\.rda$", sprintf("_%s.csv", suffix), rda_file)
+                file.rename(file.path(folder_path, rda_file), file.path(folder_path, rda_file_renamed))
+              }
+              else {
+                csv_file <- gsub("\\.rda$", ".csv", rda_file)
+                rda_file_renamed <- rda_file
+              }
+              write.csv(data, file = file.path(folder_path, csv_file), row.names = FALSE)
+
+              # rename content of rda file
+              rda_var <- gsub("\\.rda$", "", rda_file_renamed)
+              print(rda_var)
+              assign(rda_var, data)
+              do.call(save, list(rda_var, file = file.path(folder_path, sprintf("%s.rda", rda_var)), compress = "bzip2"))
+
+              # create associated documentation inside R folder
+              dataset_name <- gsub("\\.rda$", "", rda_file_renamed)
+              doc_string <- paste(
+                sprintf("#' Dataset %s", dataset_name),
+                "#'",
+                sprintf("#' %s dataset", dataset_name),
+                "#'",
+                sprintf("#' @name %s", dataset_name),
+                "#' @docType data",
+                sprintf("#' @format A data frame with %s columns:", ncol(data)),
+                "#'   \\describe{",
+                paste(sapply(names(data), function(col_name) {
+                  paste(sprintf("#'     \\item{ %s }{%s}", col_name, col_name))
+                }, USE.NAMES = FALSE), collapse="\n"),
+                "#'   }",
+                "#'", sprintf("#' @source Generated from ${{ github.repository }}."),
+                "#' @references None",
+                "#'", sprintf("#' @examples\n#' data(\"%s\")", dataset_name),
+                sep = '\n',
+                sprintf("\"%s\"", dataset_name)
+              )
+
+              doc_dir <- "datasets_doc"
+              if (!file.exists(doc_dir)) {
+                dir.create(doc_dir, recursive = TRUE)
+              }
+              writeLines(doc_string, con = file.path(doc_dir, paste0(dataset_name, ".R")))
+            }
+          }
+        shell: Rscript {0}
+
+      # zip templates data
+      - name: zip artifacts data
+        if: inputs.push-templates-data
+        run: |
+          find "$GITHUB_WORKSPACE/tmp" -type f \( -name "*.rda" -o -name "*.csv" \) -exec zip -j "$GITHUB_WORKSPACE/data.zip" {} \;
+          find "$GITHUB_WORKSPACE/datasets_doc" -type f \( -name "*.R" \) -exec zip -j "$GITHUB_WORKSPACE/doc.zip" {} \;
+
+      # store templates data as artifacts
+      - name: Archive templates data
+        if: inputs.push-templates-data
+        uses: actions/upload-artifact@v3
+        with:
+          name: data_templates
+          path: ${{ github.workspace }}/data.zip
+
+      # store templates data as artifacts
+      - name: Archive doc
+        if: inputs.push-templates-data
+        uses: actions/upload-artifact@v3
+        with:
+          name: doc_templates
+          path: ${{ github.workspace }}/doc.zip
+
+      - name: Checkout repo (PR) 🛎
+        if: inputs.push-templates-data
+        uses: actions/checkout@v3
+        with:
+          ref: main
+          repository: "${{ github.repository_owner }}/${{ env.repo }}"
+          fetch-depth: 2
+          token: ${{ secrets.PHARMAVERSE_BOT }}
+
+      - name: Add safe directory
+        if: inputs.push-templates-data
+        run: |
+          git config --global --add safe.directory "${GITHUB_WORKSPACE}"
+        shell: bash
+
+      - name: Download data artifacts
+        if: inputs.push-templates-data
+        uses: actions/download-artifact@v3
+        with:
+          name: data_templates
+
+      - name: Download doc artifacts
+        if: inputs.push-templates-data
+        uses: actions/download-artifact@v3
+        with:
+          name: doc_templates
+
+      # detect diffs om templates data
+      - name: Check templates changes
+        if: inputs.push-templates-data
+        id: changes
+        run: |
+          head_data_folder=$(git ls-tree -r HEAD data) # check if data folder exists on remote main branch
+          unzip -o "${GITHUB_WORKSPACE}/data.zip" -d "${GITHUB_WORKSPACE}/inst/extdata/"
+          unzip -o "${GITHUB_WORKSPACE}/doc.zip" -d "${GITHUB_WORKSPACE}/R"
+          # move every rda files inside data folder
+          mkdir -p data
+          for rda_file in inst/extdata/*.rda; do
+            filename=$(basename "$rda_file")
+            mv "$rda_file" "data/$filename"
+          done
+          if [ -z "${head_data_folder}" ]
+          then {
+            echo "data folder not existing on main branch yet - init mode (all templates data files will be pushed)"
+            diff_files=$(ls "${GITHUB_WORKSPACE}/data")
+          } else {
+            git add data R
+            diff_files_data=$(git status data | grep 'data/.*\.rda$')
+            diff_files_doc=$(git status R | grep 'R/.*\.R$')
+            diff_files="${diff_files_data}${diff_files_doc}"
+          }
+          fi
+          if [ -z "$diff_files" ]
+          then {
+            echo "No changes detected after running templates. Stop here"
+            echo "diff=false" >> $GITHUB_OUTPUT
+          } else {
+            echo "Changes detected, new PR will be created"
+            echo "list of files with new updates: $diff_files"
+            echo "diff=true" >> $GITHUB_OUTPUT
+          }
+          fi
+        shell: bash {0}
+
+      - name: Generate man pages
+        if: inputs.push-templates-data
+        run: |
+          Rscript -e "roxygen2::roxygenize('.', roclets = c('rd', 'collate', 'namespace'))"
+          git add man
+        shell: bash {0}
+
+      - name: Update spellchecks list
+        if: inputs.push-templates-data
+        run: |
+          spelling::update_wordlist(pkg=".", confirm=FALSE)
+        shell: Rscript {0}
+
+      - name: set source-branch name
+        if: inputs.push-templates-data
+        id: branch
+        run: |
+          repo_name=$(basename "${{ github.repository }}")
+          echo "source-branch=${{ env.source-branch }}_${repo_name}@devel" >> $GITHUB_OUTPUT
+        shell: bash {0}
+
+
+      # if diff detected, push changes
+      - name: Commit and push changes in {{ env.repo }}
+        if: ${{ inputs.push-templates-data && steps.changes.outputs.diff == 'true' }}
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: templates data updates from ${{ github.repository }}
+          file_pattern: 'inst/extdata/* data/* R/* inst/WORDLIST'
+          commit_user_name: pharmaverse-bot
+          commit_user_email: 113703390+pharmaverse-bot@users.noreply.github.com
+          branch: "${{ steps.branch.outputs.source-branch }}"
+          create_branch: true
+
+      - name: Create Pull Request
+        if: ${{ inputs.push-templates-data && steps.changes.outputs.diff == 'true' }}
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.PHARMAVERSE_BOT }}
+          script: |
+            // Look for any open PRs
+            const repo_owner = "${{ github.repository_owner }}";
+            const target_repo_name = "${{ env.repo }}";
+            const result = await github.rest.pulls.list({
+              owner: repo_owner,
+              repo: target_repo_name,
+              state: "open"
+            })
+            let create_new_pr = true;
+            for (const pr of result.data) {
+              // Look for distinct PR branch name
+              if (pr.head.ref === "${{ steps.branch.outputs.source-branch }}") {
+                console.log("PR with head ref " + pr.head.ref + " already exists");
+                create_new_pr = false;
+                break;
+              }
+            }
+            // If no PR with distinguished branch name has been found
+            // create a new PR to track changes to renv.lock.
+            if (create_new_pr) {
+              console.log("Creating a new PR");
+              const result2 = await github.rest.pulls.create({
+                title: 'Templates data updates from ${{ github.repository }}',
+                owner: repo_owner,
+                repo: target_repo_name,
+                head: '${{ steps.branch.outputs.source-branch }}',
+                base: '${{ env.target-branch }}',
+                body: [
+                  'This PR has been automatically generated by ',
+                  'check-templates workflow from ${{ github.repository }}.',
+                  '\n\nPlease review the changes.'
+                ].join('')
+              });
+              // Assign reviewers to the PR
+              let rev = process.env.reviewers;
+              let reviewers_dict = JSON.parse(rev);
+              let current_repo_name_full = "${{ github.repository }}";
+              let current_repo_name = current_repo_name_full.split('/').pop();
+              const result3 = await github.rest.pulls.requestReviewers({
+                owner: repo_owner,
+                repo: target_repo_name,
+                pull_number: result2.data.number,
+                reviewers: reviewers_dict[current_repo_name]["reviewers"]
+              });
+            }
diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml
index aa3241bf..06a22894 100644
--- a/.github/workflows/spellcheck.yml
+++ b/.github/workflows/spellcheck.yml
@@ -10,6 +10,11 @@ on:
         default: '4.1'
         required: false
         type: string
+      exclude:
+        description: 'List of paths to exclude (comma seperated list)'
+        default: ''
+        required: false
+        type: string
   push:
     branches:
       - main
@@ -97,3 +102,4 @@ jobs:
         uses: insightsengineering/r-spellcheck-action@v3
         with:
           additional_options: ""
+          exclude: "${{ inputs.exclude }}"

From ed6b6e9070524e3961bdd5fe58e6a8f2728e155b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adam=20Fory=C5=9B?= <adam.forys@gmail.com>
Date: Thu, 21 Sep 2023 11:56:40 +0200
Subject: [PATCH 2/2] Replace admiral.test with pharmaversesdtm (#142)

* Replace admiral.test with pharmaversesdtm

* Automatic renv profile update.

* Automatic renv profile update.

* Automatic renv profile update.

* Propagation trigger

---------

Co-authored-by: galachad <galachad@users.noreply.github.com>
---
 .Rprofile                            |  2 +-
 .github/workflows/propagate.yml      |  2 +-
 .github/workflows/r-renv-lock.yml    | 10 +++++-----
 renv/profiles/4.1/renv/settings.json |  3 ++-
 renv/profiles/4.2/renv/settings.json |  3 ++-
 renv/profiles/4.3/renv/settings.json |  3 ++-
 vignettes/lock_and_prop.Rmd          |  2 +-
 7 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/.Rprofile b/.Rprofile
index 51ad0fdc..d059c2fc 100644
--- a/.Rprofile
+++ b/.Rprofile
@@ -19,7 +19,7 @@
       ))
     )
   )
-  packages[!(packages %in% c("admiral", "admiraldev", "admiralci", "admiral.test", getwd()))]
+  packages[!(packages %in% c("admiral", "admiraldev", "admiralci", "admiral.test", "pharmaversesdtm", getwd()))]
 }
 
 options(renv.snapshot.filter = .get_dependencies)
diff --git a/.github/workflows/propagate.yml b/.github/workflows/propagate.yml
index 144cbfee..08a1c49a 100644
--- a/.github/workflows/propagate.yml
+++ b/.github/workflows/propagate.yml
@@ -77,7 +77,7 @@ jobs:
               cicdguy
               bms63
               bundfussr
-          - name: admiral.test
+          - name: pharmaversesdtm
             target-branch: devel
             reviewers: |-
               bms63
diff --git a/.github/workflows/r-renv-lock.yml b/.github/workflows/r-renv-lock.yml
index 5822cf6e..d0f7392c 100644
--- a/.github/workflows/r-renv-lock.yml
+++ b/.github/workflows/r-renv-lock.yml
@@ -99,7 +99,7 @@ jobs:
                 ))
               )
             )
-            packages[!(packages %in% c("admiral", "admiraldev", "admiralci", "admiral.test", getwd()))]
+            packages[!(packages %in% c("admiral", "admiraldev", "admiralci", "admiral.test", "pharmaversesdtm", getwd()))]
           }
 
           options(renv.snapshot.filter = .get_dependencies)
@@ -108,7 +108,7 @@ jobs:
             loc_path <- find.package("admiraldev", lib.loc = .libPaths(), quiet = TRUE)
             suggests <-
               renv:::renv_dependencies_discover_description(loc_path, fields = "Suggests")[["Package"]]
-            suggests <- suggests[!(suggests %in% c("admiral", "admiraldev", "admiralci", "admiral.test"))]
+            suggests <- suggests[!(suggests %in% c("admiral", "admiraldev", "admiralci", "admiral.test", "pharmaversesdtm"))]
             renv::install(suggests, repos = options("repos"))
           }
 
@@ -126,12 +126,12 @@ jobs:
           renv::install("openpharma/staged.dependencies", repos = options("repos"))
 
           # Packages from stage dependencies to ignore
-          renv::settings$ignored.packages(c("admiral", "admiraldev", "admiral.test", "admiralci"))
+          renv::settings$ignored.packages(c("admiral", "admiraldev", "admiral.test", "admiralci", "pharmaversesdtm"))
 
-          renv::install("pharmaverse/admiral.test@devel", repos = options("repos"))
+          renv::install("pharmaverse/pharmaversesdtm@devel", repos = options("repos"))
           renv::install("pharmaverse/admiraldev@devel", repos = options("repos"))
           renv_install_suggests("admiraldev")
-          renv_install_suggests("admiral.test")
+          renv_install_suggests("pharmaversesdtm")
 
           # Install dependencies
           renv::settings$snapshot.type("explicit")
diff --git a/renv/profiles/4.1/renv/settings.json b/renv/profiles/4.1/renv/settings.json
index 3462f3b0..4922677e 100644
--- a/renv/profiles/4.1/renv/settings.json
+++ b/renv/profiles/4.1/renv/settings.json
@@ -5,7 +5,8 @@
     "admiral",
     "admiraldev",
     "admiral.test",
-    "admiralci"
+    "admiralci",
+    "pharmaversesdtm"
   ],
   "package.dependency.fields": [
     "Imports",
diff --git a/renv/profiles/4.2/renv/settings.json b/renv/profiles/4.2/renv/settings.json
index 3462f3b0..4922677e 100644
--- a/renv/profiles/4.2/renv/settings.json
+++ b/renv/profiles/4.2/renv/settings.json
@@ -5,7 +5,8 @@
     "admiral",
     "admiraldev",
     "admiral.test",
-    "admiralci"
+    "admiralci",
+    "pharmaversesdtm"
   ],
   "package.dependency.fields": [
     "Imports",
diff --git a/renv/profiles/4.3/renv/settings.json b/renv/profiles/4.3/renv/settings.json
index 3462f3b0..4922677e 100644
--- a/renv/profiles/4.3/renv/settings.json
+++ b/renv/profiles/4.3/renv/settings.json
@@ -5,7 +5,8 @@
     "admiral",
     "admiraldev",
     "admiral.test",
-    "admiralci"
+    "admiralci",
+    "pharmaversesdtm"
   ],
   "package.dependency.fields": [
     "Imports",
diff --git a/vignettes/lock_and_prop.Rmd b/vignettes/lock_and_prop.Rmd
index 48f5a866..6dca810e 100644
--- a/vignettes/lock_and_prop.Rmd
+++ b/vignettes/lock_and_prop.Rmd
@@ -46,7 +46,7 @@ The first step for requesting an update for a R package in the `renv.lock` file
 
 ## Propagation Process
 
-Through GitHub Actions we have developed the ability to **propagate** updated lock files with our preferred versions of R and R packages throughout the family of admiral packages.  This ensures that developers working on `{admiral.test}` and `{admiralonco}` have the same common environment as those working on `{admiraldev}` and `{admiralvaccines}`. 
+Through GitHub Actions we have developed the ability to **propagate** updated lock files with our preferred versions of R and R packages throughout the family of admiral packages.  This ensures that developers working on `{pharmaversesdtm}` and `{admiralonco}` have the same common environment as those working on `{admiraldev}` and `{admiralvaccines}`. 
 
 
 The file that does the propagation process is called [`propagate.yml`](https://github.com/pharmaverse/admiralci/blob/main/.github/workflows/propagate.yml) located in the `.github/workflows` folder of the [admiralci repository](https://github.com/pharmaverse/admiralci/).