Initial commit

Initial commit of ARM templates, SQL Scripts, and Readme instructions.
slavatrofimov · Aug 31, 2021 · 213f57d · 213f57d
1 parent 618d4c8
commit 213f57d
Show file tree

Hide file tree

Showing 43 changed files with 4,633 additions and 2 deletions.
diff --git a/ARM/data_factory.json b/ARM/data_factory.json
diff --git a/ARM/data_factory_parameters.json b/ARM/data_factory_parameters.json
@@ -0,0 +1,21 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "companyTla": {
+            "value": "GEN-UNIQUE-3"
+        },
+        "ADLS_accountKey": {
+            "value": ""
+        },
+        "deploymentType": {
+            "value": "poc"
+        },
+        "SynapseDedicatedSQLPool_connectionString": {
+            "value": "Source=mySynapseSQLPool.sql.azuresynapse.net;Initial Catalog=myDatabaseName;User ID=mySQLAdminUserName;Password=myPassword;"
+        },
+        "ADLS_url": {
+            "value": "https://<ADLS Account Name>.dfs.core.windows.net"
+        }
+    }
+}
diff --git a/ARM/synapse_workspace.json b/ARM/synapse_workspace.json
@@ -0,0 +1,204 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "companyTla": {
+      "type": "string",
+      "metadata": {
+        "description": "This is a Three Letter Acronym for your company name. 'CON' for Contoso for example."
+      }
+    },
+    "allowAllConnections": {
+      "type": "string",
+      "allowedValues": [
+        "true",
+        "false"
+      ],
+      "defaultValue": "true"
+    },
+    "deploymentType": {
+      "type": "string",
+      "defaultValue": "poc",
+      "allowedValues": [
+        "devtest",
+        "poc",
+        "prod",
+        "shared"
+      ],
+      "metadata": {
+        "description": "Specify deployment type: DevTest, POC, Prod, Shared. This will also be used in the naming convention."
+      }
+    },
+    "sqlAdministratorLogin": {
+      "type": "string",
+      "metadata": {
+        "description": "The username of the SQL Administrator"
+      }
+    },
+    "sqlAdministratorLoginPassword": {
+      "type": "securestring",
+      "metadata": {
+        "description": "The password for the SQL Administrator"
+      }
+    },
+    "sku": {
+      "type": "string",
+      "defaultValue": "DW200c",
+      "allowedValues": [
+        "DW100c",
+        "DW200c",
+        "DW300c",
+        "DW400c",
+        "DW500c",
+        "DW1000c"
+      ],
+      "metadata": {
+        "description": "Select the SKU of the SQL pool."
+      }
+    }
+  },
+  "variables": {
+    "location": "[resourceGroup().location]",
+    "synapseName": "[toLower(concat(parameters('companyTla'),parameters('deploymentType')))]",
+    "dlsName": "[toLower(concat('dls',parameters('companyTla'),parameters('deploymentType')))]",
+    "dlsFsName": "[toLower(concat(variables('dlsName'),'fs1'))]",
+    "sqlPoolName": "[toLower(concat(variables('workspaceName'),'p1'))]",
+    "workspaceName": "[toLower(concat(variables('synapseName'),'ws1'))]",
+    "metadataSync": false
+  },
+  "resources": [
+    {
+      "type": "Microsoft.Storage/storageAccounts",
+      "apiVersion": "2019-06-01",
+      "name": "[variables('dlsName')]",
+      "location": "[variables('location')]",
+      "sku": {
+        "name": "Standard_LRS"
+      },
+      "kind": "StorageV2",
+      "properties": {
+        "accessTier": "Hot",
+        "supportsHttpsTrafficOnly": true,
+        "isHnsEnabled": true
+      },
+      "resources": [
+        {
+          "name": "[concat('default/', variables('dlsFsName'))]",
+          "type": "blobServices/containers",
+          "apiVersion": "2019-06-01",
+          "dependsOn": [
+            "[variables('dlsName')]"
+          ],
+          "properties": {
+            "publicAccess": "None"
+          }
+        }
+      ]
+    },
+    {
+      "type": "Microsoft.Synapse/workspaces",
+      "apiVersion": "2019-06-01-preview",
+      "name": "[variables('workspaceName')]",
+      "location": "[variables('location')]",
+      "identity": {
+        "type": "SystemAssigned"
+      },
+      "dependsOn": [
+        "[variables('dlsName')]",
+        "[variables('dlsFsName')]"
+      ],
+      "properties": {
+        "defaultDataLakeStorage": {
+          "accountUrl": "[reference(variables('dlsName')).primaryEndpoints.dfs]",
+          "filesystem": "[variables('dlsFsName')]"
+        },
+        "sqlAdministratorLogin": "[parameters('sqlAdministratorLogin')]",
+        "sqlAdministratorLoginPassword": "[parameters('sqlAdministratorLoginPassword')]",
+        "managedVirtualNetwork": "default"
+      },
+      "resources": [
+        {
+          "condition": "[equals(parameters('allowAllConnections'),'true')]",
+          "type": "firewallrules",
+          "apiVersion": "2019-06-01-preview",
+          "name": "allowAll",
+          "location": "[variables('location')]",
+          "dependsOn": [ "[variables('workspaceName')]" ],
+          "properties": {
+            "startIpAddress": "0.0.0.0",
+            "endIpAddress": "255.255.255.255"
+          }
+        },
+        {
+          "type": "firewallrules",
+          "apiVersion": "2019-06-01-preview",
+          "name": "AllowAllWindowsAzureIps",
+          "location": "[variables('location')]",
+          "dependsOn": [ "[variables('workspaceName')]" ],
+          "properties": {
+            "startIpAddress": "0.0.0.0",
+            "endIpAddress": "0.0.0.0"
+          }
+        },
+        {
+          "type": "managedIdentitySqlControlSettings",
+          "apiVersion": "2019-06-01-preview",
+          "name": "default",
+          "location": "[variables('location')]",
+          "dependsOn": [ "[variables('workspaceName')]" ],
+          "properties": {
+            "grantSqlControlToManagedIdentity": {
+              "desiredState": "Enabled"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "type": "Microsoft.Synapse/workspaces/sqlPools",
+      "apiVersion": "2019-06-01-preview",
+      "name": "[concat(variables('workspaceName'), '/', variables('sqlPoolName'))]",
+      "location": "[variables('location')]",
+      "sku": {
+        "name": "[parameters('sku')]"
+      },
+      "dependsOn": [
+        "[variables('workspaceName')]"
+      ],
+      "properties": {
+        "createMode": "Default",
+        "collation": "SQL_Latin1_General_CP1_CI_AS"
+      },
+      "resources": [
+        {
+          "condition": "[variables('metadataSync')]",
+          "type": "metadataSync",
+          "apiVersion": "2019-06-01-preview",
+          "name": "config",
+          "location": "[variables('location')]",
+          "dependsOn": [
+            "[variables('sqlPoolName')]"
+          ],
+          "properties": {
+            "Enabled": "[variables('metadataSync')]"
+          }
+        }
+      ]
+    },
+    {
+      "scope": "[concat('Microsoft.Storage/storageAccounts/', variables('dlsName'))]",
+      "type": "Microsoft.Authorization/roleAssignments",
+      "apiVersion": "2020-04-01-preview",
+      "name": "[guid(uniqueString(variables('dlsName')))]",
+      "location": "[variables('location')]",
+      "dependsOn": [
+        "[variables('workspaceName')]"
+      ],
+      "properties": {
+        "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', 'ba92f5b4-2d11-453d-a403-e96b0029c9fe')]",
+        "principalId": "[reference(resourceId('Microsoft.Synapse/workspaces', variables('workspaceName')), '2019-06-01-preview', 'Full').identity.principalId]",
+        "principalType": "ServicePrincipal"
+      }
+    }
+  ]
+}
diff --git a/ARM/synapse_workspace_parameters.json b/ARM/synapse_workspace_parameters.json
@@ -0,0 +1,24 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "companyTla": {
+            "value": "GEN-UNIQUE-3"
+        },
+        "allowAllConnections": {
+            "value": "true"
+        },
+        "deploymentType": {
+            "value": "poc"
+        },
+        "sqlAdministratorLogin": {
+            "value": "GEN-UNIQUE-6"
+        },
+        "sqlAdministratorLoginPassword": {
+            "value": "GEN-PASSWORD"
+        },
+        "sku": {
+            "value": "DW200c"
+        }
+    }
+}
diff --git a/ETL Scripts/Initialize/000. Initialize Database - Drop All Objects.sql b/ETL Scripts/Initialize/000. Initialize Database - Drop All Objects.sql
@@ -0,0 +1,130 @@
+IF OBJECT_ID('tempdb..#Commands') IS NOT NULL
+	DROP TABLE #Commands
+IF OBJECT_ID('tempdb..#Drops') IS NOT NULL
+	DROP TABLE #Drops
+
+SET NOCOUNT ON;
+
+-- Set this to a value to only drop objects in specific schemas.
+DECLARE @OnlyInSchemas NVARCHAR(1000)
+SET @OnlyInSchemas = N'[synthea],[vocab],[omop]';
+SET @OnlyInSchemas = REPLACE(REPLACE(@OnlyInSchemas,'[', ''), ']', '')
+
+CREATE TABLE #Commands (
+    [Description]   NVARCHAR(MAX),
+    [Line]          NVARCHAR(MAX),
+	[DateAdded]		DATETIME
+) WITH (HEAP);
+
+CREATE TABLE #Drops (
+    [Type]          NVARCHAR(2),
+    [Template]      NVARCHAR(MAX)
+) WITH (HEAP);
+
+-- -- -- -- -- OBJECTS NOT ASSOCIATED WITH TABLES -- -- -- -- --
+INSERT INTO #Drops
+SELECT N'AF', N'DROP AGGREGATE $S.$O;' UNION
+SELECT N'FN', N'DROP FUNCTION $S.$O;' UNION
+SELECT N'FS', N'DROP FUNCTION $S.$O;' UNION
+SELECT N'FT', N'DROP FUNCTION $S.$O;' UNION
+SELECT N'IF', N'DROP FUNCTION $S.$O;' UNION
+SELECT N'P', N'DROP PROCEDURE $S.$O;' UNION
+SELECT N'SN', N'DROP SYNONYM $S.$O;' UNION
+SELECT N'SQ', N'DROP QUEUE $S.$O;' UNION
+SELECT N'TR', N'DROP TRIGGER $S.$O;' UNION
+SELECT N'TT', N'DROP TYPE $S.$O;' UNION
+SELECT N'TF', N'DROP FUNCTION $S.$O;';
+
+INSERT INTO #Commands
+SELECT  QUOTENAME(RTRIM([S].[name])) + '.' + QUOTENAME(RTRIM([O].[name])),
+        REPLACE(REPLACE([D].[Template], '$S', QUOTENAME(RTRIM([S].[name]))), '$O', QUOTENAME(RTRIM([O].[name]))),
+		GETDATE()
+    FROM [sys].[objects] AS [O]
+        INNER JOIN [sys].[schemas] AS [S] ON [O].[schema_id] = [S].[schema_id]
+        INNER JOIN #Drops AS [D] ON [O].[type] COLLATE Latin1_General_CS_AS = [D].[Type] COLLATE Latin1_General_CS_AS
+        WHERE [S].[name] COLLATE Latin1_General_CS_AS IN (SELECT value FROM STRING_SPLIT(@OnlyInSchemas, ',')) 
+          AND [S].[name] COLLATE Latin1_General_CS_AS <> 'sys'
+          AND [O].[is_ms_shipped] = 0;
+
+-- -- -- -- -- OBJECTS ASSOCIATED WITH TABLES -- -- -- -- --
+DELETE FROM #Drops;
+INSERT INTO #Drops
+SELECT N'C', N'ALTER TABLE $TS.$TO DROP CONSTRAINT $O;' UNION
+SELECT N'D', N'ALTER TABLE $TS.$TO DROP CONSTRAINT $O;' UNION
+SELECT N'F', N'ALTER TABLE $TS.$TO DROP CONSTRAINT $O;' UNION
+SELECT N'PK', N'ALTER TABLE $TS.$TO DROP CONSTRAINT $O;';
+
+INSERT INTO #Commands
+SELECT  QUOTENAME(RTRIM([S].[name])) + '.' + QUOTENAME(RTRIM([PO].[name])) + '::' + QUOTENAME(RTRIM([O].[name])),
+        REPLACE(REPLACE(REPLACE([D].[Template], '$TS', QUOTENAME(RTRIM([S].[name]))), '$O', QUOTENAME(RTRIM([O].[name]))), '$TO', QUOTENAME(RTRIM([PO].[name]))),
+		GETDATE()
+    FROM [sys].[objects] AS [O]
+        INNER JOIN [sys].[objects] AS [PO] ON [O].[parent_object_id] = [PO].[object_id]
+        INNER JOIN [sys].[schemas] AS [S] ON [PO].[schema_id] = [S].[schema_id]
+        INNER JOIN #Drops AS [D] ON [O].[type] COLLATE Latin1_General_CS_AS = [D].[Type] COLLATE Latin1_General_CS_AS
+        WHERE [S].[name] COLLATE Latin1_General_CS_AS IN (SELECT value FROM STRING_SPLIT(@OnlyInSchemas, ',')) 
+          AND [S].[name] COLLATE Latin1_General_CS_AS <> 'sys'
+          AND [O].[is_ms_shipped] = 0;
+
+
+-- -- -- -- -- ACTUAL DROP -- -- -- -- --
+DELETE FROM #Drops;
+INSERT INTO #Drops
+SELECT N'U', N'DROP TABLE $S.$O;' UNION
+SELECT N'V', N'DROP TABLE $S.$O;';
+
+INSERT INTO #Commands
+SELECT  QUOTENAME(RTRIM([S].[name])) + '.' + QUOTENAME(RTRIM([O].[name])),
+        REPLACE(REPLACE([D].[Template], '$S', QUOTENAME(RTRIM([S].[name]))), '$O', QUOTENAME(RTRIM([O].[name]))),
+		GETDATE()
+    FROM [sys].[objects] AS [O]
+        INNER JOIN [sys].[schemas] AS [S] ON [O].[schema_id] = [S].[schema_id]
+        INNER JOIN #Drops AS [D] ON [O].[type] COLLATE Latin1_General_CS_AS = [D].[Type] COLLATE Latin1_General_CS_AS
+        WHERE [S].[name] COLLATE Latin1_General_CS_AS IN (SELECT value FROM STRING_SPLIT(@OnlyInSchemas, ',')) 
+          AND [S].[name] COLLATE Latin1_General_CS_AS <> 'sys'
+          AND [O].[is_ms_shipped] = 0;
+
+
+--Drop Schemas
+INSERT INTO #Commands
+SELECT  QUOTENAME(RTRIM([S].[name])),
+        'DROP SCHEMA ' + QUOTENAME(RTRIM([S].[name])),
+		GETDATE()
+	FROM [sys].[schemas] AS [S]
+        WHERE [S].[name] COLLATE Latin1_General_CS_AS IN (SELECT value FROM STRING_SPLIT(@OnlyInSchemas, ',')) 
+          AND [S].[name] COLLATE Latin1_General_CS_AS <> 'sys'
+          AND [S].[schema_id] > 7;
+
+
+-- -- -- -- -- TABLES -- -- -- -- --
+DECLARE @Description NVARCHAR(MAX);
+DECLARE @Message NVARCHAR(MAX);
+DECLARE @Command NVARCHAR(MAX);
+
+
+WHILE (SELECT COUNT(*) FROM #Commands) > 0
+
+BEGIN
+
+SELECT TOP 1 
+	@Description = Description,
+	@Command = [Line]
+FROM #Commands
+ORDER BY DateAdded ASC, Description ASC
+
+
+    SET @Message = N'Dropping ' + @Description + '...';
+    PRINT @Message;
+
+    BEGIN TRY
+        EXEC sp_executesql @Command;
+    END TRY
+    BEGIN CATCH
+        SET @Message = N'Failed to drop ' + @Description + ':';
+        PRINT @Message;
+        PRINT ERROR_MESSAGE()
+    END CATCH
+
+   DELETE FROM #Commands WHERE @Description = Description
+END
+
diff --git a/ETL Scripts/Initialize/001. Create Synthea Schema.sql b/ETL Scripts/Initialize/001. Create Synthea Schema.sql
@@ -0,0 +1 @@
+CREATE SCHEMA [synthea] AUTHORIZATION DBO;