diff --git a/.github/workflows/apim-backup.yml b/.github/workflows/apim-backup.yml new file mode 100644 index 0000000..3cdb964 --- /dev/null +++ b/.github/workflows/apim-backup.yml @@ -0,0 +1,39 @@ +name: Backup_APIM + +on: + workflow_dispatch: + schedule: + - cron: "05 3 * * *" + +jobs: + backup-apim: + runs-on: ubuntu-latest + strategy: + matrix: + environment: ["dev", "prod"] + + steps: + - name: Login via Az module + uses: azure/login@v1 + with: + creds: ${{ secrets.PROD_AZURE_CREDENTIALS }} + enable-AzPSSession: true + + # Create storage account for the environment first before running: + # New-AzStorageAccount -StorageAccountName "stapimiatibackup{env}" -Location uksouth -ResourceGroupName "rg-apim-{env}" -Type 'Standard_LRS' + - name: "Backup APIM ${{ matrix.environment }} Instance" + uses: azure/powershell@v1 + env: + ApimServiceName: apim-iati-${{ matrix.environment }} + BackupName: BackupApimIati${{ matrix.environment }} + BackupResourceGroup: rg-apim-${{ matrix.environment }} + StorageName: stapimiatibackup${{ matrix.environment }} + ContainerName: backup + with: + inlineScript: | + $StorageKey = (Get-AzStorageAccountKey -ResourceGroupName ${{ env.BackupResourceGroup }} -StorageAccountName ${{ env.StorageName }})[0].Value + $StorageContext = New-AzStorageContext -StorageAccountName ${{ env.StorageName }} -StorageAccountKey $StorageKey + $DateTime = Get-Date -Format "dd_MM_yyyy_HHmm" + $BlobName = ("${{ env.ContainerName }}" + "_" + $DateTime) + Backup-AzApiManagement -ResourceGroupName ${{ env.BackupResourceGroup }} -Name ${{ env.ApimServiceName }} -StorageContext $StorageContext -TargetContainerName ${{ env.ContainerName }} -TargetBlobName $BlobName + azPSVersion: "3.1.0" diff --git a/.github/workflows/apim-ci.yml b/.github/workflows/apim-ci.yml index ffb2189..a13e37e 100644 --- a/.github/workflows/apim-ci.yml +++ b/.github/workflows/apim-ci.yml @@ -27,6 +27,8 @@ jobs: repoUrl: https://raw.githubusercontent.com/iati/apim-iati-gateway repoApimPath: service RedisConnectionString: ${{ secrets.DEV_REDIS_CONNECTION_STRING }} + ApplicationInsightsInstanceName: appi-apim + steps: - name: "Checkout GitHub Action" @@ -61,4 +63,5 @@ jobs: --parameters ApimCapacity=${{ env.ApimCapacity }} \ --parameters ApimGatewayHostname=${{ env.ApimGatewayHostname }} \ --parameters ApimDevPortalHostname=${{ env.ApimDevPortalHostname }} \ - --parameters RedisConnectionString=${{ env.RedisConnectionString }} + --parameters RedisConnectionString=${{ env.RedisConnectionString }} \ + --parameters ApplicationInsightsInstanceName=${{ env.ApplicationInsightsInstanceName }} diff --git a/.github/workflows/apim-develop.yml b/.github/workflows/apim-develop.yml index 6dbe5fd..b8093ac 100644 --- a/.github/workflows/apim-develop.yml +++ b/.github/workflows/apim-develop.yml @@ -27,6 +27,7 @@ jobs: repoBranch: ${GITHUB_REF##*/} repoApimPath: service RedisConnectionString: ${{ secrets.DEV_REDIS_CONNECTION_STRING }} + ApplicationInsightsInstanceName: appi-apim steps: - name: "Checkout GitHub Action" @@ -61,4 +62,5 @@ jobs: --parameters ApimCapacity=${{ env.ApimCapacity }} \ --parameters ApimGatewayHostname=${{ env.ApimGatewayHostname }} \ --parameters ApimDevPortalHostname=${{ env.ApimDevPortalHostname }} \ - --parameters RedisConnectionString=${{ env.RedisConnectionString }} + --parameters RedisConnectionString=${{ env.RedisConnectionString }} \ + --parameters ApplicationInsightsInstanceName=${{ env.ApplicationInsightsInstanceName }} diff --git a/.github/workflows/apim-developer-portal-sync.yml b/.github/workflows/apim-developer-portal-sync.yml new file mode 100644 index 0000000..de8c36c --- /dev/null +++ b/.github/workflows/apim-developer-portal-sync.yml @@ -0,0 +1,58 @@ +name: Sync_Developer_Portal_Dev_To_PROD + +on: + workflow_dispatch: + +jobs: + sync-developer-portal: + runs-on: ubuntu-latest + + env: + NODE_VERSION: 14 + RESOURCE_GROUP: rg-apim + SERVICE_NAME: apim-iati + SOURCE_ENV: dev + TARGET_ENV: prod + SubscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + AZURE_CREDENTIALS: ${{ secrets.PROD_AZURE_CREDENTIALS }} + gtmContainerId: ${{ secrets.GTM_CONTAINER_ID }} + + steps: + - name: "Checkout IATI/api-management-developer-portal#master" + uses: actions/checkout@v2 + with: + repository: IATI/api-management-developer-portal + ref: master + + - name: "Login to Azure" + uses: azure/login@v1.3.0 + with: + creds: ${{ env.AZURE_CREDENTIALS }} + + - name: "Setup Node ${{ env.NODE_VERSION }} Environment" + uses: actions/setup-node@v2.3.0 + with: + node-version: ${{ env.NODE_VERSION }} + + - name: "Install Dependencies with Npm" + run: npm install + + - name: "Sync Developer Portal from Dev to Prod" + working-directory: ./scripts.v3 + run: | + node ./migrate \ + --sourceSubscriptionId ${{ env.SubscriptionId }} \ + --sourceResourceGroupName ${{ env.RESOURCE_GROUP }}-${{ env.SOURCE_ENV }} \ + --sourceServiceName ${{ env.SERVICE_NAME }}-${{ env.SOURCE_ENV }} \ + --destSubscriptionId ${{ env.SubscriptionId }} \ + --destResourceGroupName ${{ env.RESOURCE_GROUP }}-${{ env.TARGET_ENV }} \ + --destServiceName ${{ env.SERVICE_NAME }}-${{ env.TARGET_ENV }} + + - name: "Update GTM config on Prod" + working-directory: ./scripts.v3 + run: | + node ./gtm \ + --subscriptionId ${{ env.SubscriptionId }} \ + --resourceGroupName ${{ env.RESOURCE_GROUP }}-${{ env.TARGET_ENV }} \ + --serviceName ${{ env.SERVICE_NAME }}-${{ env.TARGET_ENV }} \ + --gtmContainerId ${{ env.gtmContainerId }} diff --git a/.github/workflows/apim-prod.yml b/.github/workflows/apim-prod.yml index 593dae5..394135a 100644 --- a/.github/workflows/apim-prod.yml +++ b/.github/workflows/apim-prod.yml @@ -24,6 +24,8 @@ jobs: repoBranch: ${GITHUB_REF##*/} repoApimPath: service RedisConnectionString: ${{ secrets.PROD_REDIS_CONNECTION_STRING }} + ApplicationInsightsInstanceName: appi-apim + steps: - name: "Checkout GitHub Action" @@ -58,4 +60,5 @@ jobs: --parameters ApimCapacity=${{ env.ApimCapacity }} \ --parameters ApimGatewayHostname=${{ env.ApimGatewayHostname }} \ --parameters ApimDevPortalHostname=${{ env.ApimDevPortalHostname }} \ - --parameters RedisConnectionString=${{ env.RedisConnectionString }} + --parameters RedisConnectionString=${{ env.RedisConnectionString }} \ + --parameters ApplicationInsightsInstanceName=${{ env.ApplicationInsightsInstanceName }} diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..27888ea --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "extraction_templates/azure-api-management-devops-resource-kit"] + path = extraction_templates/azure-api-management-devops-resource-kit + url = git@github.com:Azure/azure-api-management-devops-resource-kit.git diff --git a/README.md b/README.md index e9d48f6..ec9ba8b 100644 --- a/README.md +++ b/README.md @@ -6,21 +6,77 @@ - [Azure CLI](https://docs.microsoft.com/en-us/dotnet/azure/install-azure-cli) - [.NET 3.1.0](https://docs.microsoft.com/en-us/dotnet/core/install/) -- [Azure/azure-api-management-devops-resource-kit](https://github.com/Azure/azure-api-management-devops-resource-kit) -- [Example Apim Devops](https://github.com/RvLabsMSFT/rvlabs-apim-devops) ## Extracting Install Extractor Tool as CLI + ```bash -git clone git@github.com:Azure/azure-api-management-devops-resource-kit.git -cd {path_to_folder}/src/APIM_ARMTemplate/apimtemplate -dotnet pack -c Release -dotnet tool install -g --add-source .\bin\Release apimtemplate -# follow instuctions to save in PATH +cd extraction_templates/azure-api-management-devops-resource-kit/src/APIM_ARMTemplate/apimtemplate +dotnet restore +dotnet run extract --extractorConfig ../../../../apimExtract.json ``` -Run extractor w/ config -```bash -apim-templates extract --extractorConfig extraction_templates/apimExtract.json +Update the Extractor Tool + +- Update the submodule to the latest commit, then do the above + +## Developer Portal + +The developer portal look and feel customisation cannot be managed with ARM templates in source control. + +It can be synced from Dev to PROD using a node script that utilises the Management API behind the scenes. + +This is set up in a GitHub Actions Workflow that can be run manually from GitHub + +[Workflow On GitHub](https://github.com/IATI/apim-iati-gateway/actions/workflows/apim-developer-portal-sync.yml) > Run Workflow + +You can then check that the Developer Portal has been Published in the Portal [here](https://portal.azure.com/#@iatitech.onmicrosoft.com/resource/subscriptions/bcaf7a00-7a14-4932-ac41-7bb0dee0d2a9/resourceGroups/rg-apim-PROD/providers/Microsoft.ApiManagement/service/apim-iati-PROD/apim-portal) + +The PROD_AZURE_CREDENTIALS Service Principal has been given Contributor role on both the Dev and Prod Resources so that it can move the resources between them. + +## Backup + +The APIM instance can be manually backed up by running the `apim-backup.yml` GitHub Actions workflow from GitHub. It is also set to backup the dev and prod instances nightly. + +The backup is stored in a blob storage account in the same resource group as the APIM instance that it's backing up. + +## Restore + +Budget approximately 2-3hrs to complete a restore to a NEW Apim instance. + +A backup can be [restored](https://docs.microsoft.com/en-us/powershell/module/az.apimanagement/restore-azapimanagement?view=azps-6.2.1) to a NEW apim instance with the following steps: + +- [Install Azure Powershell](https://docs.microsoft.com/en-us/powershell/azure/install-az-ps?view=azps-6.2.1) +- Create a new APIM instance, e.g. `apim-iati-dr` in the appropriate resource group `rg-apim-dr` + - The SKU of the service being restored into must match the SKU of the backed-up service being restored. + - Timing: ~45min +- Create KeyVault Access Policies for the new APIM instance, so it can access `kv-iati-PROD` +- Use the below PowerShell commands to restore from the appropriate backup: + - Example below: source backup is in a storage account `stapimiatibackupprod` in resource group `rg-apim-prod`, target apim instance is `apim-iati-dr` in resource group `rg-apim-dr` +- Change the CNAME records (`developer.iatistandard.org` and `api.iatistandard.org`) in Cloudflare to point to the new APIM instance URL +- Add the Custom Domain in the Azure Portal (doesn't seem to be copied with the backup) + - Restore operation doesn't change custom hostname configuration of the target service. + - Timing: ~20min +- Copy the Developer Portal content from `dev` to your new instance (modify TARGET_ENV in `apim-developer-portal-sync.yml`) + +If just restoring to the existing `apim-iati-prod` instance, then only the PowerShell restore command would likely be neccessary. + +```pwsh +PS >$storageKey = (Get-AzStorageAccountKey -ResourceGroupName "rg-apim-prod" -StorageAccountName "stapimiatibackupprod")[0].Value + +PS >$storageContext = New-AzStorageContext -StorageAccountName "stapimiatibackupprod" -StorageAccountKey $storageKey + +PS >Restore-AzApiManagement -ResourceGroupName "rg-apim-dr" -Name "apim-iati-dr" -StorageContext $StorageContext -SourceContainerName "backup" -SourceBlobName "backup_29_07_2021_1535" ``` + +Take note of considerations [here](https://docs.microsoft.com/en-us/azure/api-management/api-management-howto-disaster-recovery-backup-restore#constraints-when-making-backup-or-restore-request) + +- Restore is a long running operation that may take up to 30 or more minutes to complete. +- [Constraints](https://docs.microsoft.com/en-us/azure/api-management/api-management-howto-disaster-recovery-backup-restore#constraints-when-making-backup-or-restore-request) +- [What is not backed up](https://docs.microsoft.com/en-us/azure/api-management/api-management-howto-disaster-recovery-backup-restore#what-is-not-backed-up) + +# Resources + +- [Azure/azure-api-management-devops-resource-kit](https://github.com/Azure/azure-api-management-devops-resource-kit) +- [Example Apim Devops](https://github.com/RvLabsMSFT/rvlabs-apim-devops) diff --git a/extraction_templates/apimExtract.json b/extraction_templates/apimExtract.json index 968eb89..ad20ba7 100644 --- a/extraction_templates/apimExtract.json +++ b/extraction_templates/apimExtract.json @@ -2,7 +2,7 @@ "sourceApimName": "apim-iati-dev", "destinationApimName": "apim-iati-dev", "resourceGroup": "rg-apim-dev", - "fileFolder": "./service", + "fileFolder": "../../../../../service", "linkedTemplatesBaseUrl": "https://raw.githubusercontent.com/IATI/apim-iati-gateway/main/service/", "policyXMLBaseUrl": "https://raw.githubusercontent.com/IATI/apim-iati-gateway/main/service/policies/", "splitAPIs": "false", diff --git a/extraction_templates/azure-api-management-devops-resource-kit b/extraction_templates/azure-api-management-devops-resource-kit new file mode 160000 index 0000000..7911271 --- /dev/null +++ b/extraction_templates/azure-api-management-devops-resource-kit @@ -0,0 +1 @@ +Subproject commit 79112711b878bcedad9d5778849bbc466f522223 diff --git a/service/apim-iati-dev-loggers.template.json b/service/apim-iati-dev-loggers.template.json new file mode 100644 index 0000000..b4c3696 --- /dev/null +++ b/service/apim-iati-dev-loggers.template.json @@ -0,0 +1,63 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "ApimServiceName": { + "type": "string" + }, + "Environment": { + "type": "string" + }, + "ApplicationInsightsInstanceName": { + "type": "string" + } + }, + "variables": { + "ApplicationInsightsInstanceNameWithEnv": "[concat(parameters('ApplicationInsightsInstanceName'), '-', parameters('Environment'))]" + }, + "resources": [ + { + "name": "[variables('ApplicationInsightsInstanceNameWithEnv')]", + "type": "Microsoft.Insights/components", + "apiVersion": "2015-05-01", + "location": "UK South", + "tags": {}, + "kind": "other", + "properties": { + "Application_Type": "other" + } + }, + { + "properties": { + "loggerType": "applicationInsights", + "description": "Application Insights logger for APIM", + "credentials": { + "instrumentationKey": "[reference(resourceId('Microsoft.Insights/components', variables('ApplicationInsightsInstanceNameWithEnv')), '2015-05-01').InstrumentationKey]" + }, + "isBuffered": true, + "resourceId": "[resourceId('Microsoft.Insights/components',variables('ApplicationInsightsInstanceNameWithEnv'))]" + }, + "name": "[concat(parameters('ApimServiceName'), '/', variables('ApplicationInsightsInstanceNameWithEnv'))]", + "type": "Microsoft.ApiManagement/service/loggers", + "apiVersion": "2021-01-01-preview" + }, + { + "properties": { + "alwaysLog": "allErrors", + "loggerId": "[concat('/loggers/', variables('ApplicationInsightsInstanceNameWithEnv'))]", + "httpCorrelationProtocol": "Legacy", + "logClientIp": true, + "sampling": { + "samplingType": "fixed", + "percentage": 100.0 + } + }, + "name": "[concat(parameters('ApimServiceName'), '/applicationinsights')]", + "type": "Microsoft.ApiManagement/service/diagnostics", + "apiVersion": "2021-01-01-preview", + "dependsOn": [ + "[resourceId('Microsoft.ApiManagement/service/loggers', parameters('ApimServiceName'), variables('ApplicationInsightsInstanceNameWithEnv'))]" + ] + } + ] +} diff --git a/service/apim-iati-dev-master.template.json b/service/apim-iati-dev-master.template.json index 899c256..7e14fab 100644 --- a/service/apim-iati-dev-master.template.json +++ b/service/apim-iati-dev-master.template.json @@ -50,22 +50,10 @@ "description": "Service url for each Api" } }, - "NamedValues": { - "type": "object", - "metadata": { - "description": "Named values" - } - }, - "ApiLoggerId": { - "type": "object", - "metadata": { - "description": "LoggerId for this api" - } - }, - "LoggerResourceId": { - "type": "object", + "ApplicationInsightsInstanceName": { + "type": "string", "metadata": { - "description": "ResourceId for the logger" + "description": "App Insights name for this apim instance (e.g. appi-apim)" } }, "SubscriptionId": { @@ -209,6 +197,32 @@ "[resourceId('Microsoft.Resources/deployments', 'apimTemplate')]" ] }, + { + "properties": { + "mode": "Incremental", + "templateLink": { + "uri": "[concat(parameters('LinkedTemplatesBaseUrl'), '/apim-iati-dev-loggers.template.json')]", + "contentVersion": "1.0.0.0" + }, + "parameters": { + "ApimServiceName": { + "value": "[parameters('ApimServiceName')]" + }, + "Environment": { + "value": "[parameters('Environment')]" + }, + "ApplicationInsightsInstanceName": { + "value": "[parameters('ApplicationInsightsInstanceName')]" + } + } + }, + "name": "loggersTemplate", + "type": "Microsoft.Resources/deployments", + "apiVersion": "2018-05-01", + "dependsOn": [ + "[resourceId('Microsoft.Resources/deployments', 'apimTemplate')]" + ] + }, { "properties": { "mode": "Incremental", diff --git a/service/apim-iati-dev-parameters.json b/service/apim-iati-dev-parameters.json index 4ccbca6..8a6f351 100644 --- a/service/apim-iati-dev-parameters.json +++ b/service/apim-iati-dev-parameters.json @@ -33,19 +33,6 @@ "validatorservicesprivate": null } }, - "NamedValues": { - "value": { - "ApimDevPortalHostname": "dev-developer.iatistandard.org", - "Environment": "dev", - "UrlEnvPrefix": "dev-" - } - }, - "ApiLoggerId": { - "value": {} - }, - "LoggerResourceId": { - "value": {} - }, "ApimSKU": { "value": "Developer" }, @@ -54,6 +41,9 @@ }, "RedisConnectionString": { "value": "placeholderstring" + }, + "ApplicationInsightsInstanceName": { + "value": "appi-apim" } } }