From 4b5fcd0a73dc483f61d8c55e70d277f9f4718215 Mon Sep 17 00:00:00 2001 From: Jackson Hoffart Date: Thu, 30 May 2024 09:22:50 +0200 Subject: [PATCH 1/5] add azure-deploy json params --- azure-deploy.json | 222 +++++++++++++++++++++++++ azure-deploy.rmi-pacta.parameters.json | 53 ++++++ 2 files changed, 275 insertions(+) create mode 100644 azure-deploy.json create mode 100644 azure-deploy.rmi-pacta.parameters.json diff --git a/azure-deploy.json b/azure-deploy.json new file mode 100644 index 0000000..385cf60 --- /dev/null +++ b/azure-deploy.json @@ -0,0 +1,222 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "0.0.0.5", + "parameters": { + "containerGroupPrefix": { + "type": "string", + "defaultValue": "workflow.benchmark.preparation", + "metadata": { + "description": "The name of the container group." + } + }, + "identity": { + "type": "string", + "metadata": { + "description": "The ID of the user assigned identity to use for the container group." + } + }, + "location": { + "type": "string", + "defaultValue": "[resourceGroup().location]", + "metadata": { + "description": "Location for all resources." + } + }, + "restartPolicy": { + "type": "string", + "defaultValue": "Never", + "allowedValues": [ + "Always", + "Never", + "OnFailure" + ], + "metadata": { + "description": "The behavior of Azure runtime if container has stopped." + } + }, + "starttime": { + "type": "string", + "defaultValue": "[utcNow()]", + "metadata": { + "description": "The time this template is deployed." + } + }, + "imageTag": { + "type": "string", + "defaultValue": "main", + "metadata": { + "description": "Image tag for the loader container." + } + }, + "logWorkspaceId": { + "type": "string", + "metadata": { + "description": "The ID for a Log Analytics Workspace." + } + }, + "logWorkspaceKey": { + "type": "securestring", + "metadata": { + "description": "The key for a Log Analytics Workspace." + } + }, + "storageAccountKeySources": { + "type": "securestring", + "metadata": { + "description": "The storage account key for the storage account for source files." + } + }, + "storageAccountNameSources": { + "type": "string", + "metadata": { + "description": "The storage account name for the storage account for source files." + } + }, + "storageAccountShareSources": { + "type": "string", + "metadata": { + "description": "The file share name for the source files." + } + }, + "storageAccountKeyOutputs": { + "type": "securestring", + "metadata": { + "description": "The storage account key for the storage account for output files." + } + }, + "storageAccountNameOutputs": { + "type": "string", + "metadata": { + "description": "The storage account name for the storage account for output files." + } + }, + "storageAccountShareOutputs": { + "type": "string", + "metadata": { + "description": "The file share name for the output files." + } + }, + "rConfigActive": { + "type": "string", + "allowedValues": [ + "2022Q4", + "2023Q4" + ], + "metadata": { + "description": "Active configuration from config.yml" + } + } + }, + "variables": { + "containerregistry": "ghcr.io/rmi-pacta", + "machineCpuCoresLimit": 1, + "machineCpuCoresRequest": 1, + "machineMemoryInGBLimit": 4, + "machineMemoryInGBRequest": 4, + "mountPathSources": "/mnt/benchmarks", + "mountPathOutputs": "/mnt/workflow-benchmark-preparation-outputs", + "containerGroupName": "[concat(parameters('containerGroupPrefix'), '-', parameters('rConfigActive'))]" + }, + "functions": [], + "resources": [ + { + "type": "Microsoft.ContainerInstance/containerGroups", + "apiVersion": "2021-09-01", + "name": "[variables('containerGroupName')]", + "location": "[parameters('location')]", + "identity": { + "type": "UserAssigned", + "userAssignedIdentities": { + "[parameters('identity')]": {} + } + }, + "properties": { + "diagnostics": { + "logAnalytics": { + "logType": "ContainerInstanceLogs", + "workspaceId": "[parameters('logWorkspaceId')]", + "workspaceKey": "[parameters('logWorkspaceKey')]" + } + }, + "containers": [ + { + "name": "benchmark-prep-runner", + "properties": { + "image": "[concat(variables('containerregistry'),'/workflow.scenario.preparation:', parameters('imageTag'))]", + "ports": [], + "resources": { + "limits": { + "cpu": "[variables('machineCpuCoresLimit')]", + "memoryInGB": "[variables('machineMemoryInGBLimit')]" + }, + "requests": { + "cpu": "[variables('machineCpuCoresRequest')]", + "memoryInGB": "[variables('machineMemoryInGBRequest')]" + } + }, + "environmentVariables": [ + { + "name": "DEPLOY_START_TIME", + "value": "[parameters('starttime')]" + }, + { + "name": "MACHINE_CORES", + "value": "[variables('machineCpuCoresRequest')]" + }, + { + "name": "LOG_LEVEL", + "value": "TRACE" + }, + { + "name": "BENCHMARKS_PREPARATION_INPUTS_PATH", + "value": "[variables('mountPathSources')]" + }, + { + "name": "BENCHMARKS_PREPARATION_OUTPUTS_PATH", + "value": "[variables('mountPathOutputs')]" + }, + { + "name": "R_CONFIG_ACTIVE", + "value": "[parameters('rConfigActive')]" + } + ], + "volumeMounts": [ + { + "name": "benchmarks", + "mountPath": "[variables('mountPathSources')]" + }, + { + "name": "benchmark-preparation-outputs", + "mountPath": "[variables('mountPathOutputs')]" + } + ] + } + } + ], + "restartPolicy": "[parameters('restartPolicy')]", + "osType": "Linux", + "volumes": [ + { + "name": "benchmarks", + "azureFile": { + "shareName": "[parameters('storageAccountShareSources')]", + "readOnly": true, + "storageAccountName": "[parameters('storageAccountNameSources')]", + "storageAccountKey": "[parameters('storageAccountKeySources')]" + } + }, + { + "name": "benchmark-preparation-outputs", + "azureFile": { + "shareName": "[parameters('storageAccountShareOutputs')]", + "readOnly": false, + "storageAccountName": "[parameters('storageAccountNameOutputs')]", + "storageAccountKey": "[parameters('storageAccountKeyOutputs')]" + } + } + ] + } + } + ], + "outputs": {} +} diff --git a/azure-deploy.rmi-pacta.parameters.json b/azure-deploy.rmi-pacta.parameters.json new file mode 100644 index 0000000..3758e4a --- /dev/null +++ b/azure-deploy.rmi-pacta.parameters.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "identity": { + "value": "/subscriptions/feef729b-4584-44af-a0f9-4827075512f9/resourceGroups/RMI-SP-PACTA-DEV/providers/Microsoft.ManagedIdentity/userAssignedIdentities/pacta-runner-dev" + }, + "storageAccountKeySources": { + "reference": { + "keyVault": { + "id": "/subscriptions/feef729b-4584-44af-a0f9-4827075512f9/resourceGroups/RMI-SP-PACTA-DEV/providers/Microsoft.KeyVault/vaults/pacta-vault-dev" + }, + "secretName": "rawdata-storageaccountkey" + } + }, + "storageAccountNameSources": { + "value": "pactarawdata" + }, + "storageAccountShareSources": { + "value": "benchmarks" + }, + "storageAccountKeyOutputs": { + "reference": { + "keyVault": { + "id": "/subscriptions/feef729b-4584-44af-a0f9-4827075512f9/resourceGroups/RMI-SP-PACTA-DEV/providers/Microsoft.KeyVault/vaults/pacta-vault-dev" + }, + "secretName": "pactadatadev-storageaccountkey" + } + }, + "storageAccountNameOutputs": { + "value": "pactadatadev" + }, + "storageAccountShareOutputs": { + "value": "workflow-benchmark-preparation-outputs" + }, + "logWorkspaceId": { + "reference": { + "keyVault": { + "id": "/subscriptions/feef729b-4584-44af-a0f9-4827075512f9/resourceGroups/RMI-SP-PACTA-DEV/providers/Microsoft.KeyVault/vaults/pacta-vault-dev" + }, + "secretName": "LogWorkspaceID-Dev" + } + }, + "logWorkspaceKey": { + "reference": { + "keyVault": { + "id": "/subscriptions/feef729b-4584-44af-a0f9-4827075512f9/resourceGroups/RMI-SP-PACTA-DEV/providers/Microsoft.KeyVault/vaults/pacta-vault-dev" + }, + "secretName": "LogWorkspaceKey-Dev" + } + } + } +} From d5f98b8fbfa8b725b07f9c91fd83176bce776f71 Mon Sep 17 00:00:00 2001 From: Jackson Hoffart Date: Thu, 30 May 2024 09:30:11 +0200 Subject: [PATCH 2/5] add run-benchmark-prep job to docker.yml --- .github/workflows/docker.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1dc1e65..c246bf5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -27,3 +27,9 @@ jobs: ] do-lint: false do-check-r-sysdeps: false + run-benchmark-prep: + uses: ./.github/workflows/run-scenario-preparation.yml + needs: [docker] + secrets: inherit + with: + image-tag: ${{ needs.docker.outputs.full-image-name }} From 1457568d440ee421558c7930c47ed6074f57a657 Mon Sep 17 00:00:00 2001 From: Jackson Hoffart Date: Thu, 30 May 2024 09:31:36 +0200 Subject: [PATCH 3/5] fix typo --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c246bf5..0887d6f 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -28,7 +28,7 @@ jobs: do-lint: false do-check-r-sysdeps: false run-benchmark-prep: - uses: ./.github/workflows/run-scenario-preparation.yml + uses: ./.github/workflows/run-benchmark-preparation.yml needs: [docker] secrets: inherit with: From 400b7cc26758745cf0d1525020d2d24b0602fcd7 Mon Sep 17 00:00:00 2001 From: Jackson Hoffart Date: Thu, 30 May 2024 13:52:38 +0200 Subject: [PATCH 4/5] fix typo --- .github/workflows/run-benchmark-preparation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-benchmark-preparation.yml b/.github/workflows/run-benchmark-preparation.yml index 2a9d610..85854d8 100644 --- a/.github/workflows/run-benchmark-preparation.yml +++ b/.github/workflows/run-benchmark-preparation.yml @@ -61,7 +61,7 @@ jobs: az --version IMAGE_TAG=$(echo $FULL_IMAGE_NAME | sed 's/^.*://') az deployment group create \ - --name "workflow.scenario.preparation-$R_CONFIG_ACTIVE" \ + --name "workflow.benchmark.preparation-$R_CONFIG_ACTIVE" \ --resource-group "$RESOURCEGROUP" \ --template-file azure-deploy.json \ --parameters azure-deploy.rmi-pacta.parameters.json \ From ad78b3e86675dfaa7b147ba831cbf258e9dd6ce7 Mon Sep 17 00:00:00 2001 From: CJ Yetman Date: Thu, 30 May 2024 15:21:15 +0200 Subject: [PATCH 5/5] Update azure-deploy.json Co-authored-by: Alex Axthelm --- azure-deploy.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-deploy.json b/azure-deploy.json index 385cf60..8c4973a 100644 --- a/azure-deploy.json +++ b/azure-deploy.json @@ -142,7 +142,7 @@ { "name": "benchmark-prep-runner", "properties": { - "image": "[concat(variables('containerregistry'),'/workflow.scenario.preparation:', parameters('imageTag'))]", + "image": "[concat(variables('containerregistry'),'/workflow.benchmark.preparation:', parameters('imageTag'))]", "ports": [], "resources": { "limits": {