Merge branch 'datahub-project:master' into master

datahub-web-react/src/app/ingest/source/builder/RecipeForm/__tests__/utils.test.ts

@@ -0,0 +1,26 @@
+import { validateURL } from '../../../utils';
+
+describe('validateURL function', () => {
+    it('should resolve if the URL is valid', async () => {
+        const validator = validateURL('test');
+        await expect(validator.validator(null, 'https://example.com')).resolves.toBeUndefined();
+        await expect(validator.validator(null, 'http://example.com')).resolves.toBeUndefined();
+        await expect(validator.validator(null, 'http://subdomain.example.com/path')).resolves.toBeUndefined();
+    });
+
+    it('should reject if the URL is invalid', async () => {
+        const validator = validateURL('test url');
+        await expect(validator.validator(null, 'http://example')).rejects.toThrowError('A valid test url is required.');
+        await expect(validator.validator(null, 'example')).rejects.toThrowError('A valid test url is required.');
+        await expect(validator.validator(null, 'http://example')).rejects.toThrowError(
+            'A valid test url is required.',
+        );
+    });
+
+    it('should resolve if the value is empty', async () => {
+        const validator = validateURL('test');
+        await expect(validator.validator(null, '')).resolves.toBeUndefined();
+        await expect(validator.validator(null, undefined)).resolves.toBeUndefined();
+        await expect(validator.validator(null, null)).resolves.toBeUndefined();
+    });
+});

datahub-web-react/src/app/ingest/source/builder/RecipeForm/azure.ts

@@ -0,0 +1,171 @@
+import { validateURL } from '../../utils';
+import { RecipeField, FieldType, setListValuesOnRecipe } from './common';
+
+export const AZURE_CLIENT_ID: RecipeField = {
+    name: 'client_id',
+    label: 'Client ID',
+    tooltip: 'Application ID. Found in your app registration on Azure AD Portal',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.client_id',
+    placeholder: '00000000-0000-0000-0000-000000000000',
+    required: true,
+    rules: null,
+};
+
+export const AZURE_TENANT_ID: RecipeField = {
+    name: 'tenant_id',
+    label: 'Tenant ID',
+    tooltip: 'Directory ID. Found in your app registration on Azure AD Portal',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.tenant_id',
+    placeholder: '00000000-0000-0000-0000-000000000000',
+    required: true,
+    rules: null,
+};
+
+export const AZURE_CLIENT_SECRET: RecipeField = {
+    name: 'client_secret',
+    label: 'Client Secret',
+    tooltip: 'The Azure client secret.',
+    type: FieldType.SECRET,
+    fieldPath: 'source.config.client_secret',
+    placeholder: '00000000-0000-0000-0000-000000000000',
+    required: true,
+    rules: null,
+};
+
+export const AZURE_REDIRECT_URL: RecipeField = {
+    name: 'redirect',
+    label: 'Redirect URL',
+    tooltip: 'Redirect URL. Found in your app registration on Azure AD Portal.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.redirect',
+    placeholder: 'https://login.microsoftonline.com/common/oauth2/nativeclient',
+    required: true,
+    rules: [() => validateURL('Redirect URI')],
+};
+
+export const AZURE_AUTHORITY_URL: RecipeField = {
+    name: 'authority',
+    label: 'Authority URL',
+    tooltip: 'Is a URL that indicates a directory that MSAL can request tokens from..',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.authority',
+    placeholder: 'https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000',
+    required: true,
+    rules: [() => validateURL('Azure authority URL')],
+};
+
+export const AZURE_TOKEN_URL: RecipeField = {
+    name: 'token_url',
+    label: 'Token URL',
+    tooltip:
+        'The token URL that acquires a token from Azure AD for authorizing requests. This source will only work with v1.0 endpoint.',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.token_url',
+    placeholder: 'https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/token',
+    required: true,
+    rules: [() => validateURL('Azure token URL')],
+};
+
+export const AZURE_GRAPH_URL: RecipeField = {
+    name: 'graph_url',
+    label: 'Graph URL',
+    tooltip: 'Microsoft Graph API endpoint',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.graph_url',
+    placeholder: 'https://graph.microsoft.com/v1.0',
+    required: true,
+    rules: [() => validateURL('Graph url URL')],
+};
+
+export const AZURE_INGEST_USERS: RecipeField = {
+    name: 'ingest_users',
+    label: 'Ingest Users',
+    tooltip: 'Flag to determine whether to ingest users from Azure AD or not.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.ingest_users',
+    rules: null,
+};
+
+export const AZURE_INGEST_GROUPS: RecipeField = {
+    name: 'ingest_groups',
+    label: 'Ingest Groups',
+    tooltip: 'Flag to determine whether to ingest groups from Azure AD or not.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.ingest_groups',
+    rules: null,
+};
+
+const schemaAllowFieldPathGroup = 'source.config.groups_pattern.allow';
+export const GROUP_ALLOW: RecipeField = {
+    name: 'groups.allow',
+    label: 'Allow Patterns',
+    tooltip:
+        'Only include specific schemas by providing the name of a schema, or a regular expression (regex) to include specific schemas. If not provided, all schemas inside allowed databases will be included.',
+    placeholder: 'group_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaAllowFieldPathGroup,
+    rules: null,
+    section: 'Group',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaAllowFieldPathGroup),
+};
+
+const schemaDenyFieldPathGroup = 'source.config.groups_pattern.deny';
+export const GROUP_DENY: RecipeField = {
+    name: 'groups.deny',
+    label: 'Deny Patterns',
+    tooltip:
+        'Exclude specific schemas by providing the name of a schema, or a regular expression (regex). If not provided, all schemas inside allowed databases will be included. Deny patterns always take precedence over allow patterns.',
+    placeholder: 'user_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaDenyFieldPathGroup,
+    rules: null,
+    section: 'Group',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaDenyFieldPathGroup),
+};
+
+const schemaAllowFieldPathUser = 'source.config.users_pattern.allow';
+export const USER_ALLOW: RecipeField = {
+    name: 'user.allow',
+    label: 'Allow Patterns',
+    tooltip:
+        'Exclude specific schemas by providing the name of a schema, or a regular expression (regex). If not provided, all schemas inside allowed databases will be included. Deny patterns always take precedence over allow patterns.',
+    placeholder: 'user_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaAllowFieldPathUser,
+    rules: null,
+    section: 'User',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaAllowFieldPathUser),
+};
+
+const schemaDenyFieldPathUser = 'source.config.users_pattern.deny';
+export const USER_DENY: RecipeField = {
+    name: 'user.deny',
+    label: 'Deny Patterns',
+    tooltip:
+        'Exclude specific schemas by providing the name of a schema, or a regular expression (regex). If not provided, all schemas inside allowed databases will be included. Deny patterns always take precedence over allow patterns.',
+    placeholder: 'user_pattern',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: schemaDenyFieldPathUser,
+    rules: null,
+    section: 'User',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, schemaDenyFieldPathUser),
+};
+
+export const SKIP_USERS_WITHOUT_GROUP: RecipeField = {
+    name: 'skip_users_without_a_group',
+    label: 'Skip users without group',
+    tooltip: 'Whether to skip users without group from Okta.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.skip_users_without_a_group',
+    rules: null,
+};

datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts

@@ -83,7 +83,7 @@ import {
     PROJECT_NAME,
 } from './lookml';
 import { PRESTO, PRESTO_HOST_PORT, PRESTO_DATABASE, PRESTO_USERNAME, PRESTO_PASSWORD } from './presto';
-import { BIGQUERY_BETA, CSV, DBT_CLOUD, MYSQL, POWER_BI, UNITY_CATALOG, VERTICA } from '../constants';
+import { AZURE, BIGQUERY_BETA, CSV, DBT_CLOUD, MYSQL, OKTA, POWER_BI, UNITY_CATALOG, VERTICA } from '../constants';
 import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta';
 import { MYSQL_HOST_PORT, MYSQL_PASSWORD, MYSQL_USERNAME } from './mysql';
 import { MSSQL, MSSQL_DATABASE, MSSQL_HOST_PORT, MSSQL_PASSWORD, MSSQL_USERNAME } from './mssql';
@@ -141,6 +141,36 @@ import {
     INCLUDE_PROJECTIONS_LINEAGE,
 } from './vertica';
 import { CSV_ARRAY_DELIMITER, CSV_DELIMITER, CSV_FILE_URL, CSV_WRITE_SEMANTICS } from './csv';
+import {
+    INCLUDE_DEPROVISIONED_USERS,
+    INCLUDE_SUSPENDED_USERS,
+    INGEST_GROUPS,
+    INGEST_USERS,
+    OKTA_API_TOKEN,
+    OKTA_DOMAIN_URL,
+    POFILE_TO_GROUP,
+    POFILE_TO_GROUP_REGX_ALLOW,
+    POFILE_TO_GROUP_REGX_DENY,
+    POFILE_TO_USER,
+    POFILE_TO_USER_REGX_ALLOW,
+    POFILE_TO_USER_REGX_DENY,
+    SKIP_USERS_WITHOUT_GROUP,
+} from './okta';
+import {
+    AZURE_AUTHORITY_URL,
+    AZURE_CLIENT_ID,
+    AZURE_CLIENT_SECRET,
+    AZURE_GRAPH_URL,
+    AZURE_INGEST_GROUPS,
+    AZURE_INGEST_USERS,
+    AZURE_REDIRECT_URL,
+    AZURE_TENANT_ID,
+    AZURE_TOKEN_URL,
+    GROUP_ALLOW,
+    GROUP_DENY,
+    USER_ALLOW,
+    USER_DENY,
+} from './azure';
 
 export enum RecipeSections {
     Connection = 0,
@@ -459,6 +489,36 @@ export const RECIPE_FIELDS: RecipeFields = {
         filterFields: [],
         advancedFields: [CSV_ARRAY_DELIMITER, CSV_DELIMITER, CSV_WRITE_SEMANTICS],
     },
+    [OKTA]: {
+        fields: [OKTA_DOMAIN_URL, OKTA_API_TOKEN, POFILE_TO_USER, POFILE_TO_GROUP],
+        filterFields: [
+            POFILE_TO_USER_REGX_ALLOW,
+            POFILE_TO_USER_REGX_DENY,
+            POFILE_TO_GROUP_REGX_ALLOW,
+            POFILE_TO_GROUP_REGX_DENY,
+        ],
+        advancedFields: [
+            INGEST_USERS,
+            INGEST_GROUPS,
+            INCLUDE_DEPROVISIONED_USERS,
+            INCLUDE_SUSPENDED_USERS,
+            STATEFUL_INGESTION_ENABLED,
+            SKIP_USERS_WITHOUT_GROUP,
+        ],
+    },
+    [AZURE]: {
+        fields: [
+            AZURE_CLIENT_ID,
+            AZURE_TENANT_ID,
+            AZURE_CLIENT_SECRET,
+            AZURE_REDIRECT_URL,
+            AZURE_AUTHORITY_URL,
+            AZURE_TOKEN_URL,
+            AZURE_GRAPH_URL,
+        ],
+        filterFields: [GROUP_ALLOW, GROUP_DENY, USER_ALLOW, USER_DENY],
+        advancedFields: [AZURE_INGEST_USERS, AZURE_INGEST_GROUPS, STATEFUL_INGESTION_ENABLED, SKIP_USERS_WITHOUT_GROUP],
+    },
 };
 
 export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS));

datahub-web-react/src/app/ingest/source/builder/RecipeForm/csv.ts

@@ -1,18 +1,6 @@
+import { validateURL } from '../../utils';
 import { RecipeField, FieldType } from './common';
 
-const validateURL = (fieldName) => {
-    return {
-        validator(_, value) {
-            const URLPattern = new RegExp(/^(?:http(s)?:\/\/)?[\w.-]+(?:\.[\w.-]+)+[\w\-._~:/?#[\]@!$&'()*+,;=.]+$/);
-            const isURLValid = URLPattern.test(value);
-            if (!value || isURLValid) {
-                return Promise.resolve();
-            }
-            return Promise.reject(new Error(`A valid ${fieldName} is required.`));
-        },
-    };
-};
-
 export const CSV_FILE_URL: RecipeField = {
     name: 'filename',
     label: 'File URL',