From bf3144e54f48fdb50de6e8bdba050e738227edde Mon Sep 17 00:00:00 2001 From: Thomas Yu Date: Sat, 28 Oct 2023 20:06:40 -0700 Subject: [PATCH] Add in integrations --- .github/workflows/ci.yaml | 8 +++--- .sqlfluffignore | 3 -- admin/integrations.sql | 60 +++++++++++++++++++++++++++++++++++++++ admin/oauth.sql | 17 ----------- admin/recover_setup.sql | 35 +++++++++-------------- admin/synapse_setup.sql | 29 ++++--------------- admin/users.sql | 20 ------------- 7 files changed, 82 insertions(+), 90 deletions(-) create mode 100644 admin/integrations.sql delete mode 100644 admin/oauth.sql diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 33428757..93483eaa 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -48,13 +48,13 @@ jobs: run: | ~/bin/snowsql -f admin/databases.sql - - name: Create tableau oauth + - name: Create users run: | - ~/bin/snowsql -f admin/oauth.sql + ~/bin/snowsql -f admin/users.sql - - name: Create users + - name: Create integration run: | - ~/bin/snowsql -f admin/users.sql --variable saml2_issuer=$saml2_issuer --variable saml2_sso_url=$saml2_sso_url --variable saml2_x509_cert=$saml2_x509_cert + ~/bin/snowsql -f admin/integrations.sql --variable saml2_issuer=$saml2_issuer --variable saml2_sso_url=$saml2_sso_url --variable saml2_x509_cert=$saml2_x509_cert # - name: Governance # run: | diff --git a/.sqlfluffignore b/.sqlfluffignore index f193ba82..69feb401 100644 --- a/.sqlfluffignore +++ b/.sqlfluffignore @@ -1,4 +1 @@ -oauth.sql policies.sql -users.sql -synapse_setup.sql diff --git a/admin/integrations.sql b/admin/integrations.sql new file mode 100644 index 00000000..b929ac77 --- /dev/null +++ b/admin/integrations.sql @@ -0,0 +1,60 @@ +!set variable_substitution=true; --noqa: PRS + +USE ROLE account_admin; + +-- * Integration to prod (SNOW-14) +CREATE STORAGE INTEGRATION IF NOT EXISTS synapse_prod_warehouse_s3 + TYPE = EXTERNAL_STAGE + STORAGE_PROVIDER = 'S3' + ENABLED = TRUE + STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::325565585839:role/snowflake-accesss-SnowflakeServiceRole-HL66JOP7K4BT' + STORAGE_ALLOWED_LOCATIONS = ('s3://prod.datawarehouse.sagebase.org'); + +-- DESC INTEGRATION synapse_prod_warehouse_s3; +CREATE STORAGE INTEGRATION IF NOT EXISTS synapse_dev_warehouse_s3 + TYPE = EXTERNAL_STAGE + STORAGE_PROVIDER = 'S3' + ENABLED = TRUE + STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::449435941126:role/test-snowflake-access-SnowflakeServiceRole-1LXZYAMMKTHJY' + STORAGE_ALLOWED_LOCATIONS = ('s3://dev.datawarehouse.sagebase.org'); +-- DESC INTEGRATION synapse_dev_warehouse_s3; + +-- RECOVER dev integration +CREATE STORAGE INTEGRATION IF NOT EXISTS recover_dev_s3 + TYPE = EXTERNAL_STAGE + STORAGE_PROVIDER = 'S3' + ENABLED = TRUE + STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::914833433684:role/snowflake_access' + STORAGE_ALLOWED_LOCATIONS = ('s3://recover-dev-processed-data', 's3://recover-dev-intermediate-data'); + +-- https://docs.snowflake.com/en/user-guide/oauth-partner +-- Integration with tableau +CREATE SECURITY INTEGRATION IF NOT EXISTS ts_oauth_int2 + TYPE = OAUTH + ENABLED = TRUE + OAUTH_CLIENT = TABLEAU_SERVER + OAUTH_REFRESH_TOKEN_VALIDITY = 86400; + +CREATE SECURITY INTEGRATION IF NOT EXISTS td_oauth_int2 + TYPE = OAUTH + ENABLED = TRUE + OAUTH_REFRESH_TOKEN_VALIDITY = 36000 + OAUTH_CLIENT = TABLEAU_DESKTOP; + +-- DESC SECURITY INTEGRATION ts_oauth_int2; +// Used these instructions to create google SAML integration +// https://community.snowflake.com/s/article/configuring-g-suite-as-an-identity-provider +create security integration IF NOT EXISTS GOOGLE_SSO + type = saml2 + enabled = true + saml2_issuer = '&saml2_issuer' + saml2_sso_url = '&saml2_sso_url' + saml2_provider = 'custom' + saml2_x509_cert='&saml2_x509_cert' + saml2_sp_initiated_login_page_label = 'GOOGLE_SSO' + saml2_enable_sp_initiated = true + SAML2_SIGN_REQUEST = true + SAML2_SNOWFLAKE_ACS_URL = 'https://mqzfhld-vp00034.snowflakecomputing.com/fed/login' + SAML2_SNOWFLAKE_ISSUER_URL = 'https://mqzfhld-vp00034.snowflakecomputing.com'; + +-- DESC security integration GOOGLE_SSO; diff --git a/admin/oauth.sql b/admin/oauth.sql deleted file mode 100644 index cb729a5a..00000000 --- a/admin/oauth.sql +++ /dev/null @@ -1,17 +0,0 @@ --- https://docs.snowflake.com/en/user-guide/oauth-partner -USE ROLE ACCOUNTADMIN; - -CREATE SECURITY INTEGRATION IF NOT EXISTS ts_oauth_int2 - TYPE = OAUTH - ENABLED = TRUE - OAUTH_CLIENT = TABLEAU_SERVER - OAUTH_REFRESH_TOKEN_VALIDITY = 86400; - -CREATE SECURITY INTEGRATION IF NOT EXISTS td_oauth_int2 - TYPE = OAUTH - ENABLED = TRUE - OAUTH_REFRESH_TOKEN_VALIDITY = 36000 - OAUTH_CLIENT = TABLEAU_DESKTOP; - - --- DESC SECURITY INTEGRATION ts_oauth_int2; diff --git a/admin/recover_setup.sql b/admin/recover_setup.sql index 586945e5..8226206b 100644 --- a/admin/recover_setup.sql +++ b/admin/recover_setup.sql @@ -1,10 +1,10 @@ CREATE DATABASE IF NOT EXISTS recover; CREATE SCHEMA IF NOT EXISTS pilot_raw - WITH MANAGED ACCESS; +WITH MANAGED ACCESS; USE SCHEMA recover.pilot_raw; USE ROLE securityadmin; -GRANT CREATE SCHEMA, USAGE ON DATABASE RECOVER +GRANT CREATE SCHEMA, USAGE ON DATABASE recover TO ROLE recover_data_engineer; GRANT ALL PRIVILEGES ON FUTURE SCHEMAS IN DATABASE recover TO ROLE recover_data_engineer; @@ -12,7 +12,7 @@ GRANT ALL PRIVILEGES ON FUTURE TABLES IN DATABASE recover TO ROLE recover_data_engineer; GRANT USAGE ON WAREHOUSE recover_xsmall TO ROLE recover_data_engineer; -GRANT USAGE ON DATABASE RECOVER +GRANT USAGE ON DATABASE recover TO ROLE recover_data_analytics; GRANT USAGE ON FUTURE SCHEMAS IN DATABASE recover TO ROLE recover_data_analytics; @@ -20,31 +20,22 @@ GRANT SELECT ON FUTURE TABLES IN DATABASE recover TO ROLE recover_data_analytics; -- Set up storage integration -use role accountadmin; - -CREATE STORAGE INTEGRATION IF NOT EXISTS recover_dev_s3 - TYPE = EXTERNAL_STAGE - STORAGE_PROVIDER = 'S3' - ENABLED = TRUE - STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::914833433684:role/snowflake_access' - STORAGE_ALLOWED_LOCATIONS = ('s3://recover-dev-processed-data', 's3://recover-dev-intermediate-data'); - DESC INTEGRATION recover_dev_s3; GRANT USAGE ON INTEGRATION recover_dev_s3 -TO ROLE SYSADMIN; -use role sysadmin; +TO ROLE sysadmin; +USE ROLE sysadmin; CREATE STAGE IF NOT EXISTS recover_dev - STORAGE_INTEGRATION = recover_dev_s3 - URL = 's3://recover-dev-processed-data' - FILE_FORMAT = (TYPE = PARQUET COMPRESSION = AUTO); + STORAGE_INTEGRATION = recover_dev_s3 --noqa: LT02,PRS + URL = 's3://recover-dev-processed-data' --noqa: LT02 + FILE_FORMAT = (TYPE = PARQUET COMPRESSION = AUTO); --noqa: LT02 CREATE STAGE IF NOT EXISTS recover_dev_intermediate - STORAGE_INTEGRATION = recover_dev_s3 - URL = 's3://recover-dev-intermediate-data' - FILE_FORMAT = (TYPE = JSON COMPRESSION = AUTO); + STORAGE_INTEGRATION = recover_dev_s3 --noqa: LT02 + URL = 's3://recover-dev-intermediate-data' --noqa: LT02 + FILE_FORMAT = (TYPE = JSON COMPRESSION = AUTO); --noqa: LT02 -LIST @recover_dev/main/parquet -PATTERN = '^((?!archive|owner).)*$'; +-- LIST @recover_dev/main/parquet +-- PATTERN = '^((?!archive|owner).)*$'; CREATE FILE FORMAT IF NOT EXISTS my_parquet TYPE = PARQUET diff --git a/admin/synapse_setup.sql b/admin/synapse_setup.sql index 4a341b93..c172b6b6 100644 --- a/admin/synapse_setup.sql +++ b/admin/synapse_setup.sql @@ -2,29 +2,17 @@ -- This script has the storage integration, external stages, and grants the resources -- to the appropriate roles USE DATABASE synapse_data_warehouse; -USE ROLE SYSADMIN; +USE ROLE sysadmin; CREATE SCHEMA IF NOT EXISTS synapse_raw - WITH MANAGED ACCESS; +WITH MANAGED ACCESS; CREATE SCHEMA IF NOT EXISTS synapse - WITH MANAGED ACCESS; +WITH MANAGED ACCESS; USE SCHEMA synapse_raw; USE WAREHOUSE compute_org; -USE ROLE account_admin; - --- * Integration to prod (SNOW-14) -CREATE STORAGE INTEGRATION IF NOT EXISTS synapse_prod_warehouse_s3 - TYPE = EXTERNAL_STAGE - STORAGE_PROVIDER = 'S3' - ENABLED = TRUE - STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::325565585839:role/snowflake-accesss-SnowflakeServiceRole-HL66JOP7K4BT' - STORAGE_ALLOWED_LOCATIONS = ('s3://prod.datawarehouse.sagebase.org'); - -DESC INTEGRATION synapse_prod_warehouse_s3; - -- * SNOW-14 CREATE STAGE IF NOT EXISTS synapse_prod_warehouse_s3_stage - STORAGE_INTEGRATION = synapse_prod_warehouse_s3 + STORAGE_INTEGRATION = synapse_prod_warehouse_s3 --noqa: LT02,PRS URL = 's3://prod.datawarehouse.sagebase.org/warehouse/' FILE_FORMAT = (TYPE = PARQUET COMPRESSION = AUTO) DIRECTORY = (ENABLE = TRUE); @@ -65,14 +53,7 @@ CREATE SCHEMA IF NOT EXISTS synapse WITH MANAGED ACCESS; USE SCHEMA synapse_raw; USE WAREHOUSE compute_org; -USE ROLE account_admin; -CREATE STORAGE INTEGRATION IF NOT EXISTS synapse_dev_warehouse_s3 - TYPE = EXTERNAL_STAGE - STORAGE_PROVIDER = 'S3' - ENABLED = TRUE - STORAGE_AWS_ROLE_ARN = 'arn:aws:iam::449435941126:role/test-snowflake-access-SnowflakeServiceRole-1LXZYAMMKTHJY' - STORAGE_ALLOWED_LOCATIONS = ('s3://dev.datawarehouse.sagebase.org'); -DESC INTEGRATION synapse_dev_warehouse_s3; + CREATE STAGE IF NOT EXISTS synapse_dev_warehouse_s3_stage STORAGE_INTEGRATION = synapse_dev_warehouse_s3 URL = 's3://dev.datawarehouse.sagebase.org/datawarehouse/' diff --git a/admin/users.sql b/admin/users.sql index 8de7b4df..e014d965 100644 --- a/admin/users.sql +++ b/admin/users.sql @@ -1,23 +1,3 @@ -!set variable_substitution=true; - -// SAML integration -use role accountadmin; -// Used these instructions to create google SAML integration -// https://community.snowflake.com/s/article/configuring-g-suite-as-an-identity-provider -create security integration IF NOT EXISTS GOOGLE_SSO - type = saml2 - enabled = true - saml2_issuer = '&saml2_issuer' - saml2_sso_url = '&saml2_sso_url' - saml2_provider = 'custom' - saml2_x509_cert='&saml2_x509_cert' - saml2_sp_initiated_login_page_label = 'GOOGLE_SSO' - saml2_enable_sp_initiated = true - SAML2_SIGN_REQUEST = true - SAML2_SNOWFLAKE_ACS_URL = 'https://mqzfhld-vp00034.snowflakecomputing.com/fed/login' - SAML2_SNOWFLAKE_ISSUER_URL = 'https://mqzfhld-vp00034.snowflakecomputing.com'; - --- DESC security integration GOOGLE_SSO; USE ROLE USERADMIN; CREATE USER IF NOT EXISTS "diep.thach@sagebase.org"; CREATE USER IF NOT EXISTS "rixing.xu@sagebase.org";