forked from datahub-project/datahub
-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'datahub-project:master' into master
- Loading branch information
Showing
36 changed files
with
2,760 additions
and
35 deletions.
There are no files selected for viewing
29 changes: 29 additions & 0 deletions
29
...ade/src/main/java/com/linkedin/datahub/upgrade/config/ReindexDomainDescriptionConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| package com.linkedin.datahub.upgrade.config; | ||
|
|
||
| import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; | ||
| import com.linkedin.datahub.upgrade.system.domaindescription.ReindexDomainDescription; | ||
| import com.linkedin.metadata.entity.AspectDao; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import org.springframework.beans.factory.annotation.Value; | ||
| import org.springframework.context.annotation.Bean; | ||
| import org.springframework.context.annotation.Conditional; | ||
| import org.springframework.context.annotation.Configuration; | ||
|
|
||
| @Configuration | ||
| @Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class) | ||
| public class ReindexDomainDescriptionConfig { | ||
|
|
||
| @Bean | ||
| public NonBlockingSystemUpgrade reindexDomainDescription( | ||
| final OperationContext opContext, | ||
| final EntityService<?> entityService, | ||
| final AspectDao aspectDao, | ||
| @Value("${systemUpdate.domainDescription.enabled}") final boolean enabled, | ||
| @Value("${systemUpdate.domainDescription.batchSize}") final Integer batchSize, | ||
| @Value("${systemUpdate.domainDescription.delayMs}") final Integer delayMs, | ||
| @Value("${systemUpdate.domainDescription.limit}") final Integer limit) { | ||
| return new ReindexDomainDescription( | ||
| opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit); | ||
| } | ||
| } |
49 changes: 49 additions & 0 deletions
49
.../java/com/linkedin/datahub/upgrade/system/domaindescription/ReindexDomainDescription.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| package com.linkedin.datahub.upgrade.system.domaindescription; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import com.linkedin.datahub.upgrade.UpgradeStep; | ||
| import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; | ||
| import com.linkedin.metadata.entity.AspectDao; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import java.util.List; | ||
| import javax.annotation.Nonnull; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| /** | ||
| * A job that reindexes all domain aspects as part of reindexing descriptions This is required to | ||
| * fix the analytics for domains | ||
| */ | ||
| @Slf4j | ||
| public class ReindexDomainDescription implements NonBlockingSystemUpgrade { | ||
|
|
||
| private final List<UpgradeStep> _steps; | ||
|
|
||
| public ReindexDomainDescription( | ||
| @Nonnull OperationContext opContext, | ||
| EntityService<?> entityService, | ||
| AspectDao aspectDao, | ||
| boolean enabled, | ||
| Integer batchSize, | ||
| Integer batchDelayMs, | ||
| Integer limit) { | ||
| if (enabled) { | ||
| _steps = | ||
| ImmutableList.of( | ||
| new ReindexDomainDescriptionStep( | ||
| opContext, entityService, aspectDao, batchSize, batchDelayMs, limit)); | ||
| } else { | ||
| _steps = ImmutableList.of(); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return this.getClass().getName(); | ||
| } | ||
|
|
||
| @Override | ||
| public List<UpgradeStep> steps() { | ||
| return _steps; | ||
| } | ||
| } |
42 changes: 42 additions & 0 deletions
42
...a/com/linkedin/datahub/upgrade/system/domaindescription/ReindexDomainDescriptionStep.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package com.linkedin.datahub.upgrade.system.domaindescription; | ||
|
|
||
| import static com.linkedin.metadata.Constants.*; | ||
|
|
||
| import com.linkedin.datahub.upgrade.system.AbstractMCLStep; | ||
| import com.linkedin.metadata.entity.AspectDao; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import javax.annotation.Nonnull; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.jetbrains.annotations.Nullable; | ||
|
|
||
| @Slf4j | ||
| public class ReindexDomainDescriptionStep extends AbstractMCLStep { | ||
|
|
||
| public ReindexDomainDescriptionStep( | ||
| OperationContext opContext, | ||
| EntityService<?> entityService, | ||
| AspectDao aspectDao, | ||
| Integer batchSize, | ||
| Integer batchDelayMs, | ||
| Integer limit) { | ||
| super(opContext, entityService, aspectDao, batchSize, batchDelayMs, limit); | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return "domain-description-v1"; | ||
| } | ||
|
|
||
| @Nonnull | ||
| @Override | ||
| protected String getAspectName() { | ||
| return DOMAIN_PROPERTIES_ASPECT_NAME; | ||
| } | ||
|
|
||
| @Nullable | ||
| @Override | ||
| protected String getUrnLike() { | ||
| return "urn:li:" + DOMAIN_ENTITY_NAME + ":%"; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
161 changes: 161 additions & 0 deletions
161
datahub-web-react/src/app/ingest/source/builder/RecipeForm/sac.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,161 @@ | ||
| import { RecipeField, FieldType, setListValuesOnRecipe } from './common'; | ||
|
|
||
| export const SAC_TENANT_URL: RecipeField = { | ||
| name: 'tenant_url', | ||
| label: 'Tenant URL', | ||
| tooltip: 'The URL of the SAP Analytics Cloud tenant.', | ||
| type: FieldType.TEXT, | ||
| fieldPath: 'source.config.tenant_url', | ||
| placeholder: 'https://company.eu10.sapanalytics.cloud', | ||
| required: true, | ||
| rules: null, | ||
| }; | ||
|
|
||
| export const SAC_TOKEN_URL: RecipeField = { | ||
| name: 'token_url', | ||
| label: 'Token URL', | ||
| tooltip: 'The OAuth 2.0 Token Service URL.', | ||
| type: FieldType.TEXT, | ||
| fieldPath: 'source.config.token_url', | ||
| placeholder: 'https://company.eu10.hana.ondemand.com/oauth/token', | ||
| required: true, | ||
| rules: null, | ||
| }; | ||
|
|
||
| export const SAC_CLIENT_ID: RecipeField = { | ||
| name: 'client_id', | ||
| label: 'Client ID', | ||
| tooltip: 'Client ID.', | ||
| type: FieldType.SECRET, | ||
| fieldPath: 'source.config.client_id', | ||
| placeholder: 'client_id', | ||
| required: true, | ||
| rules: null, | ||
| }; | ||
|
|
||
| export const SAC_CLIENT_SECRET: RecipeField = { | ||
| name: 'client_secret', | ||
| label: 'Client Secret', | ||
| tooltip: 'Client Secret.', | ||
| type: FieldType.SECRET, | ||
| fieldPath: 'source.config.client_secret', | ||
| placeholder: 'client_secret', | ||
| required: true, | ||
| rules: null, | ||
| }; | ||
|
|
||
| export const INGEST_STORIES: RecipeField = { | ||
| name: 'ingest_stories', | ||
| label: 'Ingest Stories', | ||
| tooltip: 'Whether stories should be ingested into DataHub.', | ||
| type: FieldType.BOOLEAN, | ||
| fieldPath: 'source.config.ingest_stories', | ||
| rules: null, | ||
| section: 'Stories and Applications', | ||
| }; | ||
|
|
||
| export const INGEST_APPLICATIONS: RecipeField = { | ||
| name: 'ingest_applications', | ||
| label: 'Ingest Applications', | ||
| tooltip: 'Whether applications should be ingested into DataHub.', | ||
| type: FieldType.BOOLEAN, | ||
| fieldPath: 'source.config.ingest_applications', | ||
| rules: null, | ||
| section: 'Stories and Applications', | ||
| }; | ||
|
|
||
| const resourceIdAllowFieldPath = 'source.config.resource_id_pattern.allow'; | ||
| export const RESOURCE_ID_ALLOW: RecipeField = { | ||
| name: 'resource_id_pattern.allow', | ||
| label: 'Resource Id Allow Patterns', | ||
| tooltip: | ||
| 'Only include specific Stories and Applications by providing the id of the ressource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included.', | ||
| type: FieldType.LIST, | ||
| buttonLabel: 'Add pattern', | ||
| fieldPath: resourceIdAllowFieldPath, | ||
| rules: null, | ||
| section: 'Stories and Applications', | ||
| placeholder: 'LXTH4JCE36EOYLU41PIINLYPU9XRYM26', | ||
| setValueOnRecipeOverride: (recipe: any, values: string[]) => | ||
| setListValuesOnRecipe(recipe, values, resourceIdAllowFieldPath), | ||
| }; | ||
|
|
||
| const resourceIdDenyFieldPath = 'source.config.resource_id_pattern.deny'; | ||
| export const RESOURCE_ID_DENY: RecipeField = { | ||
| name: 'resource_id_pattern.deny', | ||
| label: 'Resource Id Deny Patterns', | ||
| tooltip: | ||
| 'Exclude specific Stories and Applications by providing the id of the resource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included. Deny patterns always take precendence over Allow patterns.', | ||
| type: FieldType.LIST, | ||
| buttonLabel: 'Add pattern', | ||
| fieldPath: resourceIdDenyFieldPath, | ||
| rules: null, | ||
| section: 'Stories and Applications', | ||
| placeholder: 'LXTH4JCE36EOYLU41PIINLYPU9XRYM26', | ||
| setValueOnRecipeOverride: (recipe: any, values: string[]) => | ||
| setListValuesOnRecipe(recipe, values, resourceIdDenyFieldPath), | ||
| }; | ||
|
|
||
| const resourceNameAllowFieldPath = 'source.config.resource_id_pattern.allow'; | ||
| export const RESOURCE_NAME_ALLOW: RecipeField = { | ||
| name: 'resource_name_pattern.allow', | ||
| label: 'Resource Name Allow Patterns', | ||
| tooltip: | ||
| 'Only include specific Stories and Applications by providing the name of the ressource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included.', | ||
| type: FieldType.LIST, | ||
| buttonLabel: 'Add pattern', | ||
| fieldPath: resourceNameAllowFieldPath, | ||
| rules: null, | ||
| section: 'Stories and Applications', | ||
| placeholder: 'Name of the story', | ||
| setValueOnRecipeOverride: (recipe: any, values: string[]) => | ||
| setListValuesOnRecipe(recipe, values, resourceNameAllowFieldPath), | ||
| }; | ||
|
|
||
| const resourceNameDenyFieldPath = 'source.config.resource_name_pattern.deny'; | ||
| export const RESOURCE_NAME_DENY: RecipeField = { | ||
| name: 'resource_name_pattern.deny', | ||
| label: 'Resource Name Deny Patterns', | ||
| tooltip: | ||
| 'Exclude specific Stories and Applications by providing the name of the resource, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included. Deny patterns always take precendence over Allow patterns.', | ||
| type: FieldType.LIST, | ||
| buttonLabel: 'Add pattern', | ||
| fieldPath: resourceNameDenyFieldPath, | ||
| rules: null, | ||
| section: 'Stories and Applications', | ||
| placeholder: 'Name of the story', | ||
| setValueOnRecipeOverride: (recipe: any, values: string[]) => | ||
| setListValuesOnRecipe(recipe, values, resourceNameDenyFieldPath), | ||
| }; | ||
|
|
||
| const folderAllowFieldPath = 'source.config.resource_id_pattern.allow'; | ||
| export const FOLDER_ALLOW: RecipeField = { | ||
| name: 'folder_pattern.allow', | ||
| label: 'Folder Allow Patterns', | ||
| tooltip: | ||
| 'Only include specific Stories and Applications by providing the folder containing the resources, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included.', | ||
| type: FieldType.LIST, | ||
| buttonLabel: 'Add pattern', | ||
| fieldPath: folderAllowFieldPath, | ||
| rules: null, | ||
| section: 'Stories and Applications', | ||
| placeholder: 'Folder of the story', | ||
| setValueOnRecipeOverride: (recipe: any, values: string[]) => | ||
| setListValuesOnRecipe(recipe, values, folderAllowFieldPath), | ||
| }; | ||
|
|
||
| const folderDenyFieldPath = 'source.config.folder_pattern.deny'; | ||
| export const FOLDER_DENY: RecipeField = { | ||
| name: 'folder_pattern.deny', | ||
| label: 'Folder Deny Patterns', | ||
| tooltip: | ||
| 'Exclude specific Stories and Applications by providing the folder containing the resources, or a Regular Expression (REGEX). If not provided, all Stories and Applications will be included. Deny patterns always take precendence over Allow patterns.', | ||
| type: FieldType.LIST, | ||
| buttonLabel: 'Add pattern', | ||
| fieldPath: folderDenyFieldPath, | ||
| rules: null, | ||
| section: 'Stories and Applications', | ||
| placeholder: 'Folder of the story', | ||
| setValueOnRecipeOverride: (recipe: any, values: string[]) => | ||
| setListValuesOnRecipe(recipe, values, folderDenyFieldPath), | ||
| }; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.