Add in integrations

.github/workflows/ci.yaml

@@ -48,13 +48,13 @@ jobs:
         run: |
           ~/bin/snowsql -f admin/databases.sql
 
-      - name: Create tableau oauth
+      - name: Create users
         run: |
-          ~/bin/snowsql -f admin/oauth.sql
+          ~/bin/snowsql -f admin/users.sql
 
-      - name: Create users
+      - name: Create integration
         run: |
-          ~/bin/snowsql -f admin/users.sql --variable saml2_issuer=$saml2_issuer --variable saml2_sso_url=$saml2_sso_url --variable saml2_x509_cert=$saml2_x509_cert
+          ~/bin/snowsql -f admin/integrations.sql --variable saml2_issuer=$saml2_issuer --variable saml2_sso_url=$saml2_sso_url --variable saml2_x509_cert=$saml2_x509_cert
 
       # - name: Governance
       #   run: |

.sqlfluffignore

@@ -1,4 +1 @@
-oauth.sql
 policies.sql
-users.sql
-synapse_setup.sql

admin/integrations.sql

@@ -0,0 +1,60 @@
+!set variable_substitution=true; --noqa: PRS
+
+USE ROLE account_admin;
+
+-- * Integration to prod (SNOW-14)
+CREATE STORAGE INTEGRATION IF NOT EXISTS synapse_prod_warehouse_s3
+    TYPE = EXTERNAL_STAGE
+    STORAGE_PROVIDER = 'S3'
+    ENABLED = TRUE
+    STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::325565585839:role/snowflake-accesss-SnowflakeServiceRole-HL66JOP7K4BT'
+    STORAGE_ALLOWED_LOCATIONS = ('s3://prod.datawarehouse.sagebase.org');
+
+-- DESC INTEGRATION synapse_prod_warehouse_s3;
+CREATE STORAGE INTEGRATION IF NOT EXISTS synapse_dev_warehouse_s3
+    TYPE = EXTERNAL_STAGE
+    STORAGE_PROVIDER = 'S3'
+    ENABLED = TRUE
+    STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::449435941126:role/test-snowflake-access-SnowflakeServiceRole-1LXZYAMMKTHJY'
+    STORAGE_ALLOWED_LOCATIONS = ('s3://dev.datawarehouse.sagebase.org');
+-- DESC INTEGRATION synapse_dev_warehouse_s3;
+
+-- RECOVER dev integration
+CREATE STORAGE INTEGRATION IF NOT EXISTS recover_dev_s3
+  TYPE = EXTERNAL_STAGE
+  STORAGE_PROVIDER = 'S3'
+  ENABLED = TRUE
+  STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::914833433684:role/snowflake_access'
+  STORAGE_ALLOWED_LOCATIONS = ('s3://recover-dev-processed-data', 's3://recover-dev-intermediate-data');
+
+-- https://docs.snowflake.com/en/user-guide/oauth-partner
+-- Integration with tableau
+CREATE SECURITY INTEGRATION IF NOT EXISTS ts_oauth_int2
+  TYPE = OAUTH
+  ENABLED = TRUE
+  OAUTH_CLIENT = TABLEAU_SERVER
+  OAUTH_REFRESH_TOKEN_VALIDITY = 86400;
+
+CREATE SECURITY INTEGRATION IF NOT EXISTS td_oauth_int2
+  TYPE = OAUTH
+  ENABLED = TRUE
+  OAUTH_REFRESH_TOKEN_VALIDITY = 36000
+  OAUTH_CLIENT = TABLEAU_DESKTOP;
+
+-- DESC SECURITY INTEGRATION ts_oauth_int2;
+// Used these instructions to create google SAML integration
+// https://community.snowflake.com/s/article/configuring-g-suite-as-an-identity-provider
+create security integration IF NOT EXISTS GOOGLE_SSO
+    type = saml2
+    enabled = true
+    saml2_issuer = '&saml2_issuer'
+    saml2_sso_url = '&saml2_sso_url'
+    saml2_provider = 'custom'
+    saml2_x509_cert='&saml2_x509_cert'
+    saml2_sp_initiated_login_page_label = 'GOOGLE_SSO'
+    saml2_enable_sp_initiated = true
+    SAML2_SIGN_REQUEST = true
+    SAML2_SNOWFLAKE_ACS_URL = 'https://mqzfhld-vp00034.snowflakecomputing.com/fed/login'
+    SAML2_SNOWFLAKE_ISSUER_URL = 'https://mqzfhld-vp00034.snowflakecomputing.com';
+
+-- DESC security integration GOOGLE_SSO;

admin/recover_setup.sql

@@ -1,50 +1,41 @@
 CREATE DATABASE IF NOT EXISTS recover;
 CREATE SCHEMA IF NOT EXISTS pilot_raw
-  WITH MANAGED ACCESS;
+WITH MANAGED ACCESS;
 USE SCHEMA recover.pilot_raw;
 
 USE ROLE securityadmin;
-GRANT CREATE SCHEMA, USAGE ON DATABASE RECOVER
+GRANT CREATE SCHEMA, USAGE ON DATABASE recover
 TO ROLE recover_data_engineer;
 GRANT ALL PRIVILEGES ON FUTURE SCHEMAS IN DATABASE recover
 TO ROLE recover_data_engineer;
 GRANT ALL PRIVILEGES ON FUTURE TABLES IN DATABASE recover
 TO ROLE recover_data_engineer;
 GRANT USAGE ON WAREHOUSE recover_xsmall
 TO ROLE recover_data_engineer;
-GRANT USAGE ON DATABASE RECOVER
+GRANT USAGE ON DATABASE recover
 TO ROLE recover_data_analytics;
 GRANT USAGE ON FUTURE SCHEMAS IN DATABASE recover
 TO ROLE recover_data_analytics;
 GRANT SELECT ON FUTURE TABLES IN DATABASE recover
 TO ROLE recover_data_analytics;
 
 -- Set up storage integration
-use role accountadmin;
-
-CREATE STORAGE INTEGRATION IF NOT EXISTS recover_dev_s3
-  TYPE = EXTERNAL_STAGE
-  STORAGE_PROVIDER = 'S3'
-  ENABLED = TRUE
-  STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::914833433684:role/snowflake_access'
-  STORAGE_ALLOWED_LOCATIONS = ('s3://recover-dev-processed-data', 's3://recover-dev-intermediate-data');
-
 DESC INTEGRATION recover_dev_s3;
 GRANT USAGE ON INTEGRATION recover_dev_s3
-TO ROLE SYSADMIN;
-use role sysadmin;
+TO ROLE sysadmin;
+USE ROLE sysadmin;
 CREATE STAGE IF NOT EXISTS recover_dev
-  STORAGE_INTEGRATION = recover_dev_s3
-  URL = 's3://recover-dev-processed-data'
-  FILE_FORMAT = (TYPE = PARQUET COMPRESSION = AUTO);
+  STORAGE_INTEGRATION = recover_dev_s3 --noqa: LT02,PRS
+  URL = 's3://recover-dev-processed-data' --noqa: LT02
+  FILE_FORMAT = (TYPE = PARQUET COMPRESSION = AUTO); --noqa: LT02
 
 CREATE STAGE IF NOT EXISTS recover_dev_intermediate
-  STORAGE_INTEGRATION = recover_dev_s3
-  URL = 's3://recover-dev-intermediate-data'
-  FILE_FORMAT = (TYPE = JSON COMPRESSION = AUTO);
+  STORAGE_INTEGRATION = recover_dev_s3 --noqa: LT02
+  URL = 's3://recover-dev-intermediate-data' --noqa: LT02
+  FILE_FORMAT = (TYPE = JSON COMPRESSION = AUTO); --noqa: LT02
 
-LIST @recover_dev/main/parquet
-PATTERN = '^((?!archive|owner).)*$';
+-- LIST @recover_dev/main/parquet
+-- PATTERN = '^((?!archive|owner).)*$';
 
 CREATE FILE FORMAT IF NOT EXISTS my_parquet
   TYPE = PARQUET

admin/synapse_setup.sql

@@ -2,29 +2,17 @@
 -- This script has the storage integration, external stages, and grants the resources
 -- to the appropriate roles
 USE DATABASE synapse_data_warehouse;
-USE ROLE SYSADMIN;
+USE ROLE sysadmin;
 CREATE SCHEMA IF NOT EXISTS synapse_raw
-    WITH MANAGED ACCESS;
+WITH MANAGED ACCESS;
 CREATE SCHEMA IF NOT EXISTS synapse
-    WITH MANAGED ACCESS;
+WITH MANAGED ACCESS;
 USE SCHEMA synapse_raw;
 USE WAREHOUSE compute_org;
-USE ROLE account_admin;
-
--- * Integration to prod (SNOW-14)
-CREATE STORAGE INTEGRATION IF NOT EXISTS synapse_prod_warehouse_s3
-    TYPE = EXTERNAL_STAGE
-    STORAGE_PROVIDER = 'S3'
-    ENABLED = TRUE
-    STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::325565585839:role/snowflake-accesss-SnowflakeServiceRole-HL66JOP7K4BT'
-    STORAGE_ALLOWED_LOCATIONS = ('s3://prod.datawarehouse.sagebase.org');
-
-DESC INTEGRATION synapse_prod_warehouse_s3;
-
 
 -- * SNOW-14
 CREATE STAGE IF NOT EXISTS synapse_prod_warehouse_s3_stage
-    STORAGE_INTEGRATION = synapse_prod_warehouse_s3
+    STORAGE_INTEGRATION = synapse_prod_warehouse_s3 --noqa: LT02,PRS
     URL = 's3://prod.datawarehouse.sagebase.org/warehouse/'
     FILE_FORMAT = (TYPE = PARQUET COMPRESSION = AUTO)
     DIRECTORY = (ENABLE = TRUE);
@@ -65,14 +53,7 @@ CREATE SCHEMA IF NOT EXISTS synapse
     WITH MANAGED ACCESS;
 USE SCHEMA synapse_raw;
 USE WAREHOUSE compute_org;
-USE ROLE account_admin;
-CREATE STORAGE INTEGRATION IF NOT EXISTS synapse_dev_warehouse_s3
-    TYPE = EXTERNAL_STAGE
-    STORAGE_PROVIDER = 'S3'
-    ENABLED = TRUE
-    STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::449435941126:role/test-snowflake-access-SnowflakeServiceRole-1LXZYAMMKTHJY'
-    STORAGE_ALLOWED_LOCATIONS = ('s3://dev.datawarehouse.sagebase.org');
-DESC INTEGRATION synapse_dev_warehouse_s3;
+
 CREATE STAGE IF NOT EXISTS synapse_dev_warehouse_s3_stage
     STORAGE_INTEGRATION = synapse_dev_warehouse_s3
     URL = 's3://dev.datawarehouse.sagebase.org/datawarehouse/'

admin/users.sql

@@ -1,23 +1,3 @@
-!set variable_substitution=true;
-
-// SAML integration
-use role accountadmin;
-// Used these instructions to create google SAML integration
-// https://community.snowflake.com/s/article/configuring-g-suite-as-an-identity-provider
-create security integration IF NOT EXISTS GOOGLE_SSO
-    type = saml2
-    enabled = true
-    saml2_issuer = '&saml2_issuer'
-    saml2_sso_url = '&saml2_sso_url'
-    saml2_provider = 'custom'
-    saml2_x509_cert='&saml2_x509_cert'
-    saml2_sp_initiated_login_page_label = 'GOOGLE_SSO'
-    saml2_enable_sp_initiated = true
-    SAML2_SIGN_REQUEST = true
-    SAML2_SNOWFLAKE_ACS_URL = 'https://mqzfhld-vp00034.snowflakecomputing.com/fed/login'
-    SAML2_SNOWFLAKE_ISSUER_URL = 'https://mqzfhld-vp00034.snowflakecomputing.com';
-
--- DESC security integration GOOGLE_SSO;
 USE ROLE USERADMIN;
 CREATE USER IF NOT EXISTS "[email protected]";
 CREATE USER IF NOT EXISTS "[email protected]";