Feature/azure container

.gitignore

@@ -4,3 +4,5 @@
 .Ruserdata
 .env
 .DS_Store
+*parameters.json
+github_pat.txt

.lintr

@@ -0,0 +1,4 @@
+linters: linters_with_defaults(
+    line_length_linter = NULL,
+    object_length_linter = NULL
+  )

ACI/.gitignore

@@ -0,0 +1,3 @@
+inputs/
+outputs/
+dataprep_inputs/

ACI/Dockerfile.ACI

@@ -0,0 +1,57 @@
+FROM rocker/tidyverse:4.3.1
+
+# install system dependencies for R packages
+RUN apt-get update && apt-get install --no-install-recommends -y \
+      curl=7.81.* \
+      git=1:2.34.* \
+      gnupg=2.2.* \
+      libcurl4-openssl-dev=7.81.* \
+      libfontconfig1-dev=2.13.* \
+      libfreetype6-dev=2.11.* \
+      libfribidi-dev=1.0.* \
+      libgit2-dev=1.1.* \
+      libharfbuzz-dev=2.7.* \
+      libicu-dev=70.1-* \
+      libjpeg-dev=8c-* \
+      libpng-dev=1.6.* \
+      libssl-dev=3.0.* \
+      libtiff-dev=4.3.* \
+      libxml2-dev=2.9.* \
+      make=4.3-* \
+      pandoc=2.9.2.* \
+      zlib1g-dev=1:1.2.* \
+      && rm -rf /var/lib/apt/lists/*
+
+RUN curl -fsSL -o /tmp/google-chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \
+      && apt-get update \
+      && DEBIAN_FRONTEND='noninteractive' apt-get install --no-install-recommends -y /tmp/google-chrome.deb \
+      && rm /tmp/google-chrome.deb \
+      && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /workflow.data.preparation
+
+# set frozen CRAN repo and other R options()
+COPY ./ACI/Rprofile.site "/usr/local/lib/R/etc/Rprofile.site"
+
+# Install R dependencies
+COPY DESCRIPTION DESCRIPTION
+
+# install pak, find dependencises from DESCRIPTION, and install them.
+RUN --mount=type=secret,id=github_pat \
+    Rscript -e "\
+      Sys.setenv(GITHUB_PAT = readLines('/run/secrets/github_pat')); \
+      install.packages('pak'); \
+      deps <- pak::local_deps(root = '.'); \
+      pkg_deps <- deps[!deps[['direct']], 'ref']; \
+      cat(pkg_deps); \
+      pak::pak(pkg_deps); \
+      Sys.unsetenv('GITHUB_PAT'); \
+    "
+
+COPY ./run_pacta_data_preparation.R run_pacta_data_preparation.R
+COPY ./config.yml config.yml
+COPY ./ACI/copy_raw_data.R copy_raw_data.R
+
+COPY ./ACI/copy_files_and_run_data_prep.sh /usr/local/bin/copy_files_and_run_data_prep
+
+CMD ["copy_files_and_run_data_prep"]

ACI/RProfile.site

@@ -0,0 +1,9 @@
+options(
+  pkg.sysreqs = FALSE,
+  pkg.sysreqs_db_update = FALSE,
+  pkg.sysreqs_update = FALSE,
+  readr.show_progress = FALSE,
+  repos = c(
+    CRAN = "https://packagemanager.posit.co/cran/__linux__/jammy/2023-10-30"
+  )
+)

ACI/azure-deploy.json

@@ -0,0 +1,186 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+  "contentVersion": "0.0.0.5",
+  "parameters": {
+    "location": {
+      "type": "string",
+      "defaultValue": "[resourceGroup().location]",
+      "metadata": {
+        "description": "Location for all resources."
+      }
+    },
+    "identity": {
+      "type": "string",
+      "metadata": {
+        "description": "The ID of the user assigned identity to use for the container group."
+      }
+    },
+    "containerGroupName": {
+      "type": "string",
+      "metadata": {
+        "description": "The name of the container group."
+      }
+    },
+    "restartPolicy": {
+      "type": "string",
+      "defaultValue": "OnFailure",
+      "allowedValues": [
+        "Always",
+        "Never",
+        "OnFailure"
+      ],
+      "metadata": {
+        "description": "The behavior of Azure runtime if container has stopped."
+      }
+    },
+    "rawdata-storageaccountkey": {
+      "type": "securestring",
+      "metadata": {
+        "description": "The storage account key for the rawdata storage account."
+      }
+    },
+    "dataprepoutputs-storageaccountkey": {
+      "type": "securestring",
+      "metadata": {
+        "description": "The storage account key for the rawdata storage account."
+      }
+    },
+    "starttime": {
+      "type": "string",
+      "defaultValue": "[utcNow()]",
+      "metadata": {
+        "description": "The time to start the container group."
+      }
+    }
+  },
+  "variables": {
+    "azurecontainerregistry": "transitionmonitordockerregistry.azurecr.io"
+  },
+  "functions": [],
+  "resources": [
+    {
+      "type": "Microsoft.ContainerInstance/containerGroups",
+      "apiVersion": "2021-09-01",
+      "name": "[parameters('containerGroupName')]",
+      "location": "[parameters('location')]",
+      "identity": {
+        "type": "UserAssigned",
+        "userAssignedIdentities": {
+          "[parameters('identity')]": {}
+        }
+      },
+      "metadata": {
+        "data-prep environmentVariables description": {
+          "DEPLOY_START_TIME": "The time the container was deployed.",
+          "R_CONFIG_ACTIVE": "The active config for the container.",
+          "R_CONFIG_FILE": "The config file for the container.",
+          "LOG_LEVEL": "The log level for the container. See {rlog} docs."
+        }
+      },
+      "properties": {
+        "containers": [
+          {
+            "name": "data-prep",
+            "properties": {
+              "image": "[concat(variables('azurecontainerregistry'),'/workflow.data.preparation_aci:latest')]",
+              "ports": [],
+              "resources": {
+                "requests": {
+                  "cpu": 1,
+                  "memoryInGB": 32
+                  "gpu": {
+                    "count": 1,
+                    "sku": "V100"
+                  }
+                }
+              },
+              "environmentVariables": [
+                {
+                  "name": "DEPLOY_START_TIME",
+                  "value": "[parameters('starttime')]"
+                },
+                {
+                  "name": "R_CONFIG_ACTIVE",
+                  "value": "2022Q4_CICD"
+                },
+                {
+                  "name": "R_CONFIG_FILE",
+                  "value": "/workflow.data.preparation/config.yml"
+                },
+                {
+                  "name": "LOG_LEVEL",
+                  "value": "TRACE"
+                }
+              ],
+              "volumeMounts": [
+                {
+                  "name": "factset-extracted",
+                  "mountPath": "/mnt/factset-extracted/"
+                },
+                {
+                  "name": "rawdatavolume",
+                  "mountPath": "/mnt/rawdata/"
+                },
+                {
+                  "name": "dataprepinputsvolume",
+                  "mountPath": "/mnt/dataprep_inputs"
+                },
+                {
+                  "name": "outputsvolume",
+                  "mountPath": "/mnt/outputs/"
+                }
+              ]
+            }
+          }
+        ],
+        "imageRegistryCredentials": [
+          {
+            "server": "[variables('azurecontainerregistry')]",
+            "identity": "[parameters('identity')]"
+          }
+        ],
+        "restartPolicy": "[parameters('restartPolicy')]",
+        "osType": "Linux",
+        "volumes": [
+          {
+            "name": "factset-extracted",
+            "azureFile": {
+              "shareName": "factset-extracted",
+              "readOnly": true,
+              "storageAccountName": "pactarawdata",
+              "storageAccountKey": "[parameters('rawdata-storageaccountkey')]"
+            }
+          },
+          {
+            "name": "rawdatavolume",
+            "azureFile": {
+              "shareName": "rawdata",
+              "readOnly": true,
+              "storageAccountName": "pactarawdata",
+              "storageAccountKey": "[parameters('rawdata-storageaccountkey')]"
+            }
+          },
+          {
+            "name": "dataprepinputsvolume",
+            "azureFile": {
+              "shareName": "dataprep-inputs",
+              "readOnly": false,
+              "storageAccountName": "pactarawdata",
+              "storageAccountKey": "[parameters('rawdata-storageaccountkey')]"
+            }
+          },
+          {
+            "name": "outputsvolume",
+            "azureFile": {
+              "shareName": "data-prep-outputs",
+              "readOnly": false,
+              "storageAccountName": "pactadata",
+              "storageAccountKey": "[parameters('dataprepoutputs-storageaccountkey')]"
+            }
+          }
+        ]
+      }
+    }
+  ],
+  "outputs": {}
+}

ACI/copy_files_and_run_data_prep.sh

@@ -0,0 +1,16 @@
+#! /bin/sh
+set -e
+
+# check memory available
+free -m | cat
+
+inputs_dir="/mnt/dataprep_inputs"
+
+# copy raw data, then run normal data prep script
+Rscript /workflow.data.preparation/copy_raw_data.R 2>&1 | \
+  tee "$inputs_dir/$DEPLOY_START_TIME-copy.log"
+
+Rscript /workflow.data.preparation/run_pacta_data_preparation.R 2>&1 | \
+  tee "$inputs_dir/$DEPLOY_START_TIME-prep.log"
+
+exit 0