Skip to content

Commit

Permalink
Merge branch 'linkedin:master' into master
Browse files Browse the repository at this point in the history
  • Loading branch information
shirshanka authored Dec 10, 2021
2 parents 0249638 + d651040 commit 1f2de3b
Show file tree
Hide file tree
Showing 33 changed files with 264 additions and 111 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,8 @@


@SuppressWarnings("checkstyle:HideUtilityClassConstructor")
@SpringBootApplication(exclude = {RestClientAutoConfiguration.class}, scanBasePackages = {
"com.linkedin.gms.factory.common", "com.linkedin.gms.factory.kafka", "com.linkedin.gms.factory.search",
"com.linkedin.datahub.upgrade.config", "com.linkedin.gms.factory.entity"})
@SpringBootApplication(exclude = {RestClientAutoConfiguration.class}, scanBasePackages = {"com.linkedin.gms.factory",
"com.linkedin.datahub.upgrade.config"})
public class UpgradeCliApplication {
public static void main(String[] args) {
new SpringApplicationBuilder(UpgradeCliApplication.class, UpgradeCli.class).web(WebApplicationType.NONE).run(args);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import com.linkedin.datahub.upgrade.UpgradeStep;
import com.linkedin.datahub.upgrade.UpgradeStepResult;
import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.entity.client.RestliEntityClient;
import java.util.function.Function;
import lombok.RequiredArgsConstructor;

Expand All @@ -14,7 +14,7 @@
public class GMSDisableWriteModeStep implements UpgradeStep {

private final Authentication _systemAuthentication;
private final EntityClient _entityClient;
private final RestliEntityClient _entityClient;

@Override
public String id() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import com.linkedin.datahub.upgrade.UpgradeStep;
import com.linkedin.datahub.upgrade.UpgradeStepResult;
import com.linkedin.datahub.upgrade.impl.DefaultUpgradeStepResult;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.entity.client.RestliEntityClient;
import java.util.function.Function;
import lombok.RequiredArgsConstructor;

Expand All @@ -14,7 +14,7 @@
public class GMSEnableWriteModeStep implements UpgradeStep {

private final Authentication _systemAuthentication;
private final EntityClient _entityClient;
private final RestliEntityClient _entityClient;

@Override
public String id() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.linkedin.datahub.upgrade.UpgradeContext;
import com.linkedin.datahub.upgrade.UpgradeStep;
import com.linkedin.datahub.upgrade.UpgradeStepResult;
Expand All @@ -12,11 +13,16 @@
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
import java.util.Map;
import java.util.function.Function;
import lombok.RequiredArgsConstructor;


@RequiredArgsConstructor
public class GMSQualificationStep implements UpgradeStep {

private final Map<String, String> configToMatch;

private static String convertStreamToString(InputStream is) {

BufferedReader reader = new BufferedReader(new InputStreamReader(is));
Expand All @@ -39,8 +45,6 @@ private static String convertStreamToString(InputStream is) {
return sb.toString();
}

public GMSQualificationStep() { }

@Override
public String id() {
return "GMSQualificationStep";
Expand All @@ -51,6 +55,15 @@ public int retryCount() {
return 2;
}

private boolean isEligible(ObjectNode configJson) {
for (String key : configToMatch.keySet()) {
if (!configJson.has(key) || !configJson.get(key).asText().equals(configToMatch.get(key))) {
return false;
}
}
return true;
}

@Override
public Function<UpgradeContext, UpgradeStepResult> executable() {
return (context) -> {
Expand All @@ -65,7 +78,7 @@ public Function<UpgradeContext, UpgradeStepResult> executable() {

ObjectMapper mapper = new ObjectMapper();
JsonNode configJson = mapper.readTree(responseString);
if (configJson.get("noCode").asBoolean()) {
if (isEligible((ObjectNode) configJson)) {
return new DefaultUpgradeStepResult(
id(),
UpgradeStepResult.Result.SUCCEEDED);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import com.datahub.authentication.Authentication;
import com.linkedin.datahub.upgrade.nocode.NoCodeUpgrade;

import com.linkedin.entity.client.RestliEntityClient;
import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.models.registry.EntityRegistry;
Expand All @@ -22,7 +21,7 @@ public class NoCodeUpgradeConfig {
ApplicationContext applicationContext;

@Bean(name = "noCodeUpgrade")
@DependsOn({"ebeanServer", "entityService", "systemAuthentication", "javaEntityClient", "entityRegistry"})
@DependsOn({"ebeanServer", "entityService", "systemAuthentication", "restliEntityClient", "entityRegistry"})
@Nonnull
public NoCodeUpgrade createInstance() {
final EbeanServer ebeanServer = applicationContext.getBean(EbeanServer.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@

import com.datahub.authentication.Authentication;
import com.linkedin.datahub.upgrade.restorebackup.RestoreBackup;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.entity.client.JavaEntityClient;
import com.linkedin.entity.client.RestliEntityClient;
import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.graph.GraphService;
import com.linkedin.metadata.models.registry.EntityRegistry;
Expand All @@ -23,24 +22,19 @@ public class RestoreBackupConfig {
ApplicationContext applicationContext;

@Bean(name = "restoreBackup")
@DependsOn({"ebeanServer", "entityService", "systemAuthentication", "javaEntityClient", "graphService", "searchService", "entityRegistry"})
@DependsOn({"ebeanServer", "entityService", "systemAuthentication", "restliEntityClient", "graphService",
"searchService", "entityRegistry"})
@Nonnull
public RestoreBackup createInstance() {
final EbeanServer ebeanServer = applicationContext.getBean(EbeanServer.class);
final EntityService entityService = applicationContext.getBean(EntityService.class);
final Authentication systemAuthentication = applicationContext.getBean(Authentication.class);
final EntityClient entityClient = applicationContext.getBean(JavaEntityClient.class);
final RestliEntityClient entityClient = applicationContext.getBean(RestliEntityClient.class);
final GraphService graphClient = applicationContext.getBean(GraphService.class);
final EntitySearchService searchClient = applicationContext.getBean(EntitySearchService.class);
final EntityRegistry entityRegistry = applicationContext.getBean(EntityRegistry.class);

return new RestoreBackup(
ebeanServer,
entityService,
entityRegistry,
systemAuthentication,
entityClient,
graphClient,
searchClient);
return new RestoreBackup(ebeanServer, entityService, entityRegistry, systemAuthentication, entityClient,
graphClient, searchClient);
}
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package com.linkedin.datahub.upgrade.nocode;

import com.datahub.authentication.Authentication;
import com.google.common.collect.ImmutableMap;
import com.linkedin.datahub.upgrade.Upgrade;
import com.linkedin.datahub.upgrade.UpgradeCleanupStep;
import com.linkedin.datahub.upgrade.UpgradeStep;
import com.linkedin.datahub.upgrade.common.steps.GMSEnableWriteModeStep;
import com.linkedin.datahub.upgrade.common.steps.GMSQualificationStep;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.entity.client.RestliEntityClient;
import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.models.registry.EntityRegistry;
import io.ebean.EbeanServer;
Expand All @@ -30,7 +31,7 @@ public NoCodeUpgrade(
final EntityService entityService,
final EntityRegistry entityRegistry,
final Authentication systemAuthentication,
final EntityClient entityClient) {
final RestliEntityClient entityClient) {
_steps = buildUpgradeSteps(
server,
entityService,
Expand Down Expand Up @@ -64,10 +65,10 @@ private List<UpgradeStep> buildUpgradeSteps(
final EntityService entityService,
final EntityRegistry entityRegistry,
final Authentication systemAuthentication,
final EntityClient entityClient) {
final RestliEntityClient entityClient) {
final List<UpgradeStep> steps = new ArrayList<>();
steps.add(new RemoveAspectV2TableStep(server));
steps.add(new GMSQualificationStep());
steps.add(new GMSQualificationStep(ImmutableMap.of("noCode", "true")));
steps.add(new UpgradeQualificationStep(server));
steps.add(new CreateAspectTableStep(server));
steps.add(new DataMigrationStep(server, entityService, entityRegistry));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,7 @@
import com.linkedin.datahub.upgrade.common.steps.ClearSearchServiceStep;
import com.linkedin.datahub.upgrade.common.steps.GMSDisableWriteModeStep;
import com.linkedin.datahub.upgrade.common.steps.GMSEnableWriteModeStep;
import com.linkedin.datahub.upgrade.common.steps.GMSQualificationStep;
import com.linkedin.entity.client.EntityClient;
import com.linkedin.entity.client.RestliEntityClient;
import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.graph.GraphService;
import com.linkedin.metadata.models.registry.EntityRegistry;
Expand All @@ -29,7 +28,7 @@ public RestoreBackup(
final EntityService entityService,
final EntityRegistry entityRegistry,
final Authentication systemAuthentication,
final EntityClient entityClient,
final RestliEntityClient entityClient,
final GraphService graphClient,
final EntitySearchService searchClient) {
_steps = buildSteps(server, entityService, entityRegistry, systemAuthentication, entityClient, graphClient, searchClient);
Expand All @@ -50,11 +49,10 @@ private List<UpgradeStep> buildSteps(
final EntityService entityService,
final EntityRegistry entityRegistry,
final Authentication systemAuthentication,
final EntityClient entityClient,
final RestliEntityClient entityClient,
final GraphService graphClient,
final EntitySearchService searchClient) {
final List<UpgradeStep> steps = new ArrayList<>();
steps.add(new GMSQualificationStep());
steps.add(new GMSDisableWriteModeStep(systemAuthentication, entityClient));
steps.add(new ClearSearchServiceStep(searchClient, true));
steps.add(new ClearGraphServiceStep(graphClient, true));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import com.linkedin.datahub.upgrade.UpgradeStep;
import com.linkedin.datahub.upgrade.common.steps.ClearGraphServiceStep;
import com.linkedin.datahub.upgrade.common.steps.ClearSearchServiceStep;
import com.linkedin.datahub.upgrade.common.steps.GMSQualificationStep;
import com.linkedin.metadata.entity.EntityService;
import com.linkedin.metadata.graph.GraphService;
import com.linkedin.metadata.models.registry.EntityRegistry;
Expand Down Expand Up @@ -42,7 +41,6 @@ private List<UpgradeStep> buildSteps(final EbeanServer server, final EntityServi
final EntityRegistry entityRegistry, final EntitySearchService entitySearchService,
final GraphService graphService) {
final List<UpgradeStep> steps = new ArrayList<>();
steps.add(new GMSQualificationStep());
steps.add(new ClearSearchServiceStep(entitySearchService, false));
steps.add(new ClearGraphServiceStep(graphService, false));
steps.add(new SendMAEStep(server, entityService, entityRegistry));
Expand Down
11 changes: 10 additions & 1 deletion datahub-web-react/src/app/entity/shared/utils.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
export function urlEncodeUrn(urn: string) {
return urn && urn.replace(/%/g, '%25').replace(/\//g, '%2F').replace(/\?/g, '%3F').replace(/#/g, '%23');
return (
urn &&
urn
.replace(/%/g, '%25')
.replace(/\//g, '%2F')
.replace(/\?/g, '%3F')
.replace(/#/g, '%23')
.replace(/\[/g, '%5B')
.replace(/\]/g, '%5D')
);
}

export function notEmpty<TValue>(value: TValue | null | undefined): value is TValue {
Expand Down
11 changes: 8 additions & 3 deletions docs-website/sidebars.js
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,14 @@ module.exports = {
"docs/lineage/sample_code",
],
},
{
Guides: [
"metadata-ingestion/adding-source",
"docs/how/add-custom-ingestion-source",
"docs/how/add-custom-data-platform",
"docs/how/add-user-data",
],
},
],
"Metadata Modeling": [
"docs/modeling/metadata-model",
Expand Down Expand Up @@ -184,7 +192,6 @@ module.exports = {
// TODO: the titles of these should not be in question form in the sidebar
"docs/developers",
"docs/docker/development",
"metadata-ingestion/adding-source",
{
type: "doc",
label: "Ingesting files from S3",
Expand All @@ -206,8 +213,6 @@ module.exports = {
"docs/how/delete-metadata",
"datahub-web-react/src/app/analytics/README",
"metadata-ingestion/developing",
"docs/how/add-custom-data-platform",
"docs/how/add-custom-ingestion-source",
{
"Module READMEs": [
"datahub-web-react/README",
Expand Down
92 changes: 92 additions & 0 deletions docs/how/add-user-data.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
# Adding user in DataHub

This guide shares how you can add user metadata in DataHub. Usually you would want to use one of our sources for ingesting user metadata. But if there is no connector for your use case then you would want to use this guide.

:::note

This does not allow you add new users for Authentication. If you want to add a new user in DataHub for Login please refer to [JaaS Authentication](./auth/jaas.md)

:::

You can look at all aspects supported for users in [CorpUserAspect](../../metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/CorpUserAspect.pdl)

## Using File-Based Ingestion Recipe

Define a JSON File containing your user
```my-user.json
[
{
"auditHeader": null,
"proposedSnapshot": {
"com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": {
"urn": "urn:li:corpuser:aseem.bansal",
"aspects": [
{
"com.linkedin.pegasus2avro.identity.CorpUserInfo": {
"active": true,
"displayName": {
"string": "Aseem Bansal"
},
"email": "[email protected]",
"title": {
"string": "Software Engineer"
},
"managerUrn": null,
"departmentId": null,
"departmentName": null,
"firstName": null,
"lastName": null,
"fullName": {
"string": "Aseem Bansal"
},
"countryCode": null
}
}
]
}
}
}
]
```

Define an [ingestion recipe](https://datahubproject.io/docs/metadata-ingestion/#recipes)

```
---
# see https://datahubproject.io/docs/metadata-ingestion/source_docs/file for complete documentation
source:
type: "file"
config:
filename: "./my-user.json"
# see https://datahubproject.io/docs/metadata-ingestion/sink_docs/datahub for complete documentation
sink:
...
```

Use [DataHub CLI](../cli.md) to do the ingestion.

## Using Rest.li API

```
curl 'http://localhost:8080/entities?action=ingest' -X POST --data '{
"entity": {
"value": {
"com.linkedin.metadata.snapshot.CorpUserSnapshot": {
"urn": "urn:li:corpuser:aseem.bansal",
"aspects": [{
"com.linkedin.identity.CorpUserInfo": {
"active": true,
"displayName": "Aseem Bansal",
"email": "[email protected]",
"title": "Software Engineer",
"fullName": "Aseem Bansal"
}
}]
}
}
}
}'
```

Loading

0 comments on commit 1f2de3b

Please sign in to comment.