From b091e4615d915783818c1750599723b933d41267 Mon Sep 17 00:00:00 2001 From: skrydal Date: Wed, 11 Dec 2024 17:02:31 +0100 Subject: [PATCH 01/16] feat(ingest/kafka): Flag for optional schemas ingestion (#12077) --- docs/how/updating-datahub.md | 3 +- .../datahub/ingestion/source/kafka/kafka.py | 29 +- .../tests/integration/kafka/kafka_to_file.yml | 1 + .../kafka_without_schemas_mces_golden.json | 575 ++++++++++++++++++ .../kafka/kafka_without_schemas_to_file.yml | 16 + .../tests/integration/kafka/test_kafka.py | 9 +- .../tests/unit/test_kafka_source.py | 5 +- 7 files changed, 620 insertions(+), 18 deletions(-) create mode 100644 metadata-ingestion/tests/integration/kafka/kafka_without_schemas_mces_golden.json create mode 100644 metadata-ingestion/tests/integration/kafka/kafka_without_schemas_to_file.yml diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 8ba83768512a5f..5bc0e66fa2ff1d 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -31,7 +31,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe urn:li:dataset:(urn:li:dataPlatform:powerbi,[.]...,) ``` - The config `include_workspace_name_in_dataset_urn` is default to `false` for backward compatiblity, However, we recommend enabling this flag after performing the necessary cleanup. + The config `include_workspace_name_in_dataset_urn` is default to `false` for backward compatibility, However, we recommend enabling this flag after performing the necessary cleanup. If stateful ingestion is enabled, running ingestion with the latest CLI version will handle the cleanup automatically. Otherwise, we recommend soft deleting all powerbi data via the DataHub CLI: `datahub delete --platform powerbi --soft` and then re-ingest with the latest CLI version, ensuring the `include_workspace_name_in_dataset_urn` configuration is set to true. @@ -39,6 +39,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe - #11742: For PowerBi ingestion, `use_powerbi_email` is now enabled by default when extracting ownership information. - #12056: The DataHub Airflow plugin no longer supports Airflow 2.1 and Airflow 2.2. - #12056: The DataHub Airflow plugin now defaults to the v2 plugin implementation. +- #12077: `Kafka` source no longer ingests schemas from schema registry as separate entities by default, set `ingest_schemas_as_entities` to `true` to ingest them - OpenAPI Update: PIT Keep Alive parameter added to scroll. NOTE: This parameter requires the `pointInTimeCreationEnabled` feature flag to be enabled and the `elasticSearch.implementation` configuration to be `elasticsearch`. This feature is not supported for OpenSearch at this time and the parameter will not be respected without both of these set. - OpenAPI Update 2: Previously there was an incorrectly marked parameter named `sort` on the generic list entities endpoint for v3. This parameter is deprecated and only supports a single string value while the documentation indicates it supports a list of strings. This documentation error has been fixed and the correct field, `sortCriteria`, is now documented which supports a list of strings. diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py index 709ba431f0f87b..fa842a15ba7328 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py +++ b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka.py @@ -141,6 +141,10 @@ class KafkaSourceConfig( default=False, description="Disables the utilization of the TopicRecordNameStrategy for Schema Registry subjects. For more information, visit: https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#handling-differences-between-preregistered-and-client-derived-schemas:~:text=io.confluent.kafka.serializers.subject.TopicRecordNameStrategy", ) + ingest_schemas_as_entities: bool = pydantic.Field( + default=False, + description="Enables ingesting schemas from schema registry as separate entities, in addition to the topics", + ) def get_kafka_consumer( @@ -343,17 +347,20 @@ def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: else: self.report.report_dropped(topic) - # Get all subjects from schema registry and ingest them as SCHEMA DatasetSubTypes - for subject in self.schema_registry_client.get_subjects(): - try: - yield from self._extract_record( - subject, True, topic_detail=None, extra_topic_config=None - ) - except Exception as e: - logger.warning(f"Failed to extract subject {subject}", exc_info=True) - self.report.report_warning( - "subject", f"Exception while extracting topic {subject}: {e}" - ) + if self.source_config.ingest_schemas_as_entities: + # Get all subjects from schema registry and ingest them as SCHEMA DatasetSubTypes + for subject in self.schema_registry_client.get_subjects(): + try: + yield from self._extract_record( + subject, True, topic_detail=None, extra_topic_config=None + ) + except Exception as e: + logger.warning( + f"Failed to extract subject {subject}", exc_info=True + ) + self.report.report_warning( + "subject", f"Exception while extracting topic {subject}: {e}" + ) def _extract_record( self, diff --git a/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml b/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml index 380df845e737c6..cde21d85ed2d94 100644 --- a/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml +++ b/metadata-ingestion/tests/integration/kafka/kafka_to_file.yml @@ -3,6 +3,7 @@ run_id: kafka-test source: type: kafka config: + ingest_schemas_as_entities: true connection: bootstrap: "localhost:29092" schema_registry_url: "http://localhost:28081" diff --git a/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_mces_golden.json b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_mces_golden.json new file mode 100644 index 00000000000000..7810c8077b31d5 --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_mces_golden.json @@ -0,0 +1,575 @@ +[ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "key_topic", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "44fd7a7b325d6fdd4275b1f02a79c1a8", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "", + "keySchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}", + "keySchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=long].id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "id", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=string].namespace", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "namespace", + "recursive": false, + "isPartOfKey": true + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka" + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "Partitions": "1", + "Replication Factor": "1", + "min.insync.replicas": "1", + "retention.bytes": "-1", + "retention.ms": "604800000", + "cleanup.policy": "delete", + "max.message.bytes": "1048588", + "unclean.leader.election.enable": "false" + }, + "name": "key_topic", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Topic" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "key_value_topic", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "a79a2fe3adab60b21d272a9cc3e93595", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\",\"tags\":[\"Email\"]},{\"name\":\"firstName\",\"type\":\"string\",\"tags\":[\"Name\"]},{\"name\":\"lastName\",\"type\":\"string\",\"tags\":[\"Name\"]}],\"tags\":[\"PII\"]}", + "documentSchemaType": "AVRO", + "keySchema": "{\"type\":\"record\",\"name\":\"UserKey\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Key schema for kafka topic\",\"fields\":[{\"name\":\"id\",\"type\":\"long\"},{\"name\":\"namespace\",\"type\":\"string\"}]}", + "keySchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=long].id", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.NumberType": {} + } + }, + "nativeDataType": "id", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[key=True].[type=UserKey].[type=string].namespace", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "namespace", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "email", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Email" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Email\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].firstName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "firstName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].lastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "lastName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:PII" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "Partitions": "1", + "Replication Factor": "1", + "min.insync.replicas": "1", + "retention.bytes": "-1", + "retention.ms": "604800000", + "cleanup.policy": "delete", + "max.message.bytes": "1048588", + "unclean.leader.election.enable": "false", + "Schema Name": "key_value_topic-value" + }, + "name": "key_value_topic", + "description": "Value schema for kafka topic", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Topic" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "domains", + "aspect": { + "json": { + "domains": [ + "urn:li:domain:sales" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,key_value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "value_topic", + "platform": "urn:li:dataPlatform:kafka", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "62c7c400ec5760797a59c45e59c2f2dc", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.KafkaSchema": { + "documentSchema": "{\"type\":\"record\",\"name\":\"CreateUserRequest\",\"namespace\":\"io.codebrews.createuserrequest\",\"doc\":\"Value schema for kafka topic\",\"fields\":[{\"name\":\"email\",\"type\":\"string\",\"tags\":[\"Email\"]},{\"name\":\"firstName\",\"type\":\"string\",\"tags\":[\"Name\"]},{\"name\":\"lastName\",\"type\":\"string\",\"tags\":[\"Name\"]}],\"tags\":[\"PII\"]}", + "documentSchemaType": "AVRO", + "keySchema": "\"string\"", + "keySchemaType": "AVRO" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[key=True].[type=string]", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": true + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].email", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "email", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Email" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Email\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].firstName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "firstName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + }, + { + "fieldPath": "[version=2.0].[type=CreateUserRequest].[type=string].lastName", + "nullable": false, + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "lastName", + "recursive": false, + "globalTags": { + "tags": [ + { + "tag": "urn:li:tag:Name" + } + ] + }, + "isPartOfKey": false, + "jsonProps": "{\"tags\": [\"Name\"]}" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.common.BrowsePaths": { + "paths": [ + "/prod/kafka" + ] + } + }, + { + "com.linkedin.pegasus2avro.common.GlobalTags": { + "tags": [ + { + "tag": "urn:li:tag:PII" + } + ] + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "Partitions": "1", + "Replication Factor": "1", + "min.insync.replicas": "1", + "retention.bytes": "-1", + "retention.ms": "604800000", + "cleanup.policy": "delete", + "max.message.bytes": "1048588", + "unclean.leader.election.enable": "false", + "Schema Name": "value_topic-value" + }, + "name": "value_topic", + "description": "Value schema for kafka topic", + "tags": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "subTypes", + "aspect": { + "json": { + "typeNames": [ + "Topic" + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:kafka,value_topic,PROD)", + "changeType": "UPSERT", + "aspectName": "browsePathsV2", + "aspect": { + "json": { + "path": [] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Email", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Email" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:Name", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "Name" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +}, +{ + "entityType": "tag", + "entityUrn": "urn:li:tag:PII", + "changeType": "UPSERT", + "aspectName": "tagKey", + "aspect": { + "json": { + "name": "PII" + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "kafka-test", + "lastRunId": "no-run-id-provided" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_to_file.yml b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_to_file.yml new file mode 100644 index 00000000000000..7f44e43c3c4908 --- /dev/null +++ b/metadata-ingestion/tests/integration/kafka/kafka_without_schemas_to_file.yml @@ -0,0 +1,16 @@ +run_id: kafka-test + +source: + type: kafka + config: + connection: + bootstrap: "localhost:29092" + schema_registry_url: "http://localhost:28081" + domain: + "urn:li:domain:sales": + allow: + - "key_value_topic" +sink: + type: file + config: + filename: "./kafka_without_schemas_mces.json" diff --git a/metadata-ingestion/tests/integration/kafka/test_kafka.py b/metadata-ingestion/tests/integration/kafka/test_kafka.py index bf0ec1845a66c2..0d9a714625e96b 100644 --- a/metadata-ingestion/tests/integration/kafka/test_kafka.py +++ b/metadata-ingestion/tests/integration/kafka/test_kafka.py @@ -43,20 +43,21 @@ def mock_kafka_service(docker_compose_runner, test_resources_dir): yield docker_compose_runner +@pytest.mark.parametrize("approach", ["kafka_without_schemas", "kafka"]) @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_kafka_ingest( - mock_kafka_service, test_resources_dir, pytestconfig, tmp_path, mock_time + mock_kafka_service, test_resources_dir, pytestconfig, tmp_path, mock_time, approach ): # Run the metadata ingestion pipeline. - config_file = (test_resources_dir / "kafka_to_file.yml").resolve() + config_file = (test_resources_dir / f"{approach}_to_file.yml").resolve() run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path) # Verify the output. mce_helpers.check_golden_file( pytestconfig, - output_path=tmp_path / "kafka_mces.json", - golden_path=test_resources_dir / "kafka_mces_golden.json", + output_path=tmp_path / f"{approach}_mces.json", + golden_path=test_resources_dir / f"{approach}_mces_golden.json", ignore_paths=[], ) diff --git a/metadata-ingestion/tests/unit/test_kafka_source.py b/metadata-ingestion/tests/unit/test_kafka_source.py index cab0a2bce7ba8c..1a8afe1b956fae 100644 --- a/metadata-ingestion/tests/unit/test_kafka_source.py +++ b/metadata-ingestion/tests/unit/test_kafka_source.py @@ -330,6 +330,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: "topic2-key": "test.acryl.Topic2Key", "topic2-value": "test.acryl.Topic2Value", }, + "ingest_schemas_as_entities": True, } ctx = PipelineContext(run_id="test") kafka_source = KafkaSource.create(source_config, ctx) @@ -478,8 +479,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: kafka_source = KafkaSource.create(source_config, ctx) workunits = list(kafka_source.get_workunits()) - - assert len(workunits) == 6 + assert len(workunits) == 2 if ignore_warnings_on_schema_type: assert not kafka_source.report.warnings else: @@ -622,6 +622,7 @@ def mock_get_latest_version(subject_name: str) -> Optional[RegisteredSchema]: kafka_source = KafkaSource.create( { "connection": {"bootstrap": "localhost:9092"}, + "ingest_schemas_as_entities": True, "meta_mapping": { "owner": { "match": "^@(.*)", From f1ef4f8e5f7e4facee5d80e71c7ce6c8051bafcc Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 11 Dec 2024 13:59:14 -0500 Subject: [PATCH 02/16] feat(structuredProperties) Add new settings aspect plus graphql changes for structured props (#12052) --- .../CreateStructuredPropertyResolver.java | 119 ++++++++--- .../DeleteStructuredPropertyResolver.java | 18 ++ .../RemoveStructuredPropertiesResolver.java | 2 +- .../UpdateStructuredPropertyResolver.java | 196 +++++++++++++++--- .../UpsertStructuredPropertiesResolver.java | 2 +- .../types/chart/mappers/ChartMapper.java | 3 +- .../container/mappers/ContainerMapper.java | 4 +- .../corpgroup/mappers/CorpGroupMapper.java | 3 +- .../corpuser/mappers/CorpUserMapper.java | 3 +- .../dashboard/mappers/DashboardMapper.java | 3 +- .../dataflow/mappers/DataFlowMapper.java | 3 +- .../types/datajob/mappers/DataJobMapper.java | 3 +- .../mappers/DataProductMapper.java | 3 +- .../types/dataset/mappers/DatasetMapper.java | 3 +- .../graphql/types/domain/DomainMapper.java | 4 +- .../glossary/mappers/GlossaryNodeMapper.java | 3 +- .../glossary/mappers/GlossaryTermMapper.java | 3 +- .../mlmodel/mappers/MLFeatureMapper.java | 3 +- .../mlmodel/mappers/MLFeatureTableMapper.java | 3 +- .../mlmodel/mappers/MLModelGroupMapper.java | 3 +- .../types/mlmodel/mappers/MLModelMapper.java | 3 +- .../mlmodel/mappers/MLPrimaryKeyMapper.java | 3 +- .../types/schemafield/SchemaFieldMapper.java | 3 +- .../StructuredPropertiesMapper.java | 17 +- .../StructuredPropertyMapper.java | 18 ++ .../StructuredPropertyType.java | 3 +- .../src/main/resources/properties.graphql | 84 +++++++- .../CreateStructuredPropertyResolverTest.java | 120 ++++++++++- .../DeleteStructuredPropertyResolverTest.java | 91 ++++++++ .../StructuredPropertyUtilsTest.java | 42 ++++ .../UpdateStructuredPropertyResolverTest.java | 102 ++++++++- .../models/StructuredPropertyUtils.java | 50 +++++ .../java/com/linkedin/metadata/Constants.java | 2 + .../structuredproperties.py | 12 +- .../StructuredPropertyDefinition.pdl | 38 ++-- .../structured/StructuredPropertySettings.pdl | 64 ++++++ .../src/main/resources/entity-registry.yml | 1 + 37 files changed, 913 insertions(+), 124 deletions(-) create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolverTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/StructuredPropertyUtilsTest.java create mode 100644 metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertySettings.pdl diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java index 328f63b893d06f..7d232748f0d93c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolver.java @@ -1,7 +1,8 @@ package com.linkedin.datahub.graphql.resolvers.structuredproperties; import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; -import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_ENTITY_NAME; +import static com.linkedin.datahub.graphql.resolvers.mutate.MutationUtils.buildMetadataChangeProposalWithUrn; +import static com.linkedin.metadata.Constants.*; import com.linkedin.common.urn.Urn; import com.linkedin.data.template.SetMode; @@ -12,20 +13,24 @@ import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.CreateStructuredPropertyInput; import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.StructuredPropertySettingsInput; import com.linkedin.datahub.graphql.types.structuredproperty.StructuredPropertyMapper; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.aspect.patch.builder.StructuredPropertyDefinitionPatchBuilder; +import com.linkedin.metadata.models.StructuredPropertyUtils; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.structured.PrimitivePropertyValue; import com.linkedin.structured.PropertyCardinality; import com.linkedin.structured.PropertyValue; import com.linkedin.structured.StructuredPropertyKey; +import com.linkedin.structured.StructuredPropertySettings; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; -import java.util.UUID; import java.util.concurrent.CompletableFuture; import javax.annotation.Nonnull; @@ -54,40 +59,28 @@ public CompletableFuture get(final DataFetchingEnviron "Unable to create structured property. Please contact your admin."); } final StructuredPropertyKey key = new StructuredPropertyKey(); - final String id = input.getId() != null ? input.getId() : UUID.randomUUID().toString(); + final String id = + StructuredPropertyUtils.getPropertyId(input.getId(), input.getQualifiedName()); key.setId(id); final Urn propertyUrn = EntityKeyUtils.convertEntityKeyToUrn(key, STRUCTURED_PROPERTY_ENTITY_NAME); - StructuredPropertyDefinitionPatchBuilder builder = - new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); - - builder.setQualifiedName(input.getQualifiedName()); - builder.setValueType(input.getValueType()); - input.getEntityTypes().forEach(builder::addEntityType); - if (input.getDisplayName() != null) { - builder.setDisplayName(input.getDisplayName()); - } - if (input.getDescription() != null) { - builder.setDescription(input.getDescription()); - } - if (input.getImmutable() != null) { - builder.setImmutable(input.getImmutable()); - } - if (input.getTypeQualifier() != null) { - buildTypeQualifier(input, builder); - } - if (input.getAllowedValues() != null) { - buildAllowedValues(input, builder); + + if (_entityClient.exists(context.getOperationContext(), propertyUrn)) { + throw new IllegalArgumentException( + "A structured property already exists with this urn"); } - if (input.getCardinality() != null) { - builder.setCardinality( - PropertyCardinality.valueOf(input.getCardinality().toString())); + + List mcps = new ArrayList<>(); + + // first, create the property definition itself + mcps.add(createPropertyDefinition(context, propertyUrn, id, input)); + + // then add the settings aspect if we're adding any settings inputs + if (input.getSettings() != null) { + mcps.add(createPropertySettings(context, propertyUrn, input.getSettings())); } - builder.setCreated(context.getOperationContext().getAuditStamp()); - builder.setLastModified(context.getOperationContext().getAuditStamp()); - MetadataChangeProposal mcp = builder.build(); - _entityClient.ingestProposal(context.getOperationContext(), mcp, false); + _entityClient.batchIngestProposals(context.getOperationContext(), mcps, false); EntityResponse response = _entityClient.getV2( @@ -103,6 +96,72 @@ public CompletableFuture get(final DataFetchingEnviron }); } + private MetadataChangeProposal createPropertySettings( + @Nonnull final QueryContext context, + @Nonnull final Urn propertyUrn, + final StructuredPropertySettingsInput settingsInput) + throws Exception { + StructuredPropertySettings settings = new StructuredPropertySettings(); + + if (settingsInput.getIsHidden() != null) { + settings.setIsHidden(settingsInput.getIsHidden()); + } + if (settingsInput.getShowInSearchFilters() != null) { + settings.setShowInSearchFilters(settingsInput.getShowInSearchFilters()); + } + if (settingsInput.getShowInAssetSummary() != null) { + settings.setShowInAssetSummary(settingsInput.getShowInAssetSummary()); + } + if (settingsInput.getShowAsAssetBadge() != null) { + settings.setShowAsAssetBadge(settingsInput.getShowAsAssetBadge()); + } + if (settingsInput.getShowInColumnsTable() != null) { + settings.setShowInColumnsTable(settingsInput.getShowInColumnsTable()); + } + settings.setLastModified(context.getOperationContext().getAuditStamp()); + + StructuredPropertyUtils.validatePropertySettings(settings, true); + + return buildMetadataChangeProposalWithUrn( + propertyUrn, STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME, settings); + } + + private MetadataChangeProposal createPropertyDefinition( + @Nonnull final QueryContext context, + @Nonnull final Urn propertyUrn, + @Nonnull final String id, + final CreateStructuredPropertyInput input) + throws Exception { + StructuredPropertyDefinitionPatchBuilder builder = + new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); + + builder.setQualifiedName(id); + builder.setValueType(input.getValueType()); + input.getEntityTypes().forEach(builder::addEntityType); + if (input.getDisplayName() != null) { + builder.setDisplayName(input.getDisplayName()); + } + if (input.getDescription() != null) { + builder.setDescription(input.getDescription()); + } + if (input.getImmutable() != null) { + builder.setImmutable(input.getImmutable()); + } + if (input.getTypeQualifier() != null) { + buildTypeQualifier(input, builder); + } + if (input.getAllowedValues() != null) { + buildAllowedValues(input, builder); + } + if (input.getCardinality() != null) { + builder.setCardinality(PropertyCardinality.valueOf(input.getCardinality().toString())); + } + builder.setCreated(context.getOperationContext().getAuditStamp()); + builder.setLastModified(context.getOperationContext().getAuditStamp()); + + return builder.build(); + } + private void buildTypeQualifier( @Nonnull final CreateStructuredPropertyInput input, @Nonnull final StructuredPropertyDefinitionPatchBuilder builder) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java index e7d59494654fdd..58f8d340fcc074 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolver.java @@ -6,6 +6,7 @@ import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; import com.linkedin.datahub.graphql.exception.AuthorizationException; import com.linkedin.datahub.graphql.generated.DeleteStructuredPropertyInput; import com.linkedin.entity.client.EntityClient; @@ -42,6 +43,23 @@ public CompletableFuture get(final DataFetchingEnvironment environment) "Unable to delete structured property. Please contact your admin."); } _entityClient.deleteEntity(context.getOperationContext(), propertyUrn); + // Asynchronously Delete all references to the entity (to return quickly) + GraphQLConcurrencyUtils.supplyAsync( + () -> { + try { + _entityClient.deleteEntityReferences( + context.getOperationContext(), propertyUrn); + } catch (Exception e) { + log.error( + String.format( + "Caught exception while attempting to clear all entity references for Structured Property with urn %s", + propertyUrn), + e); + } + return null; + }, + this.getClass().getSimpleName(), + "get"); return true; } catch (Exception e) { throw new RuntimeException( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java index ea8c6dac36a4af..313e0a16d8916d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/RemoveStructuredPropertiesResolver.java @@ -93,7 +93,7 @@ public CompletableFuture get(final DataFetchingEnviron "Unable to update structured property. Please contact your admin."); } final Urn propertyUrn = UrnUtils.getUrn(input.getUrn()); - StructuredPropertyDefinition existingDefinition = + final EntityResponse entityResponse = getExistingStructuredProperty(context, propertyUrn); - StructuredPropertyDefinitionPatchBuilder builder = - new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); - if (input.getDisplayName() != null) { - builder.setDisplayName(input.getDisplayName()); - } - if (input.getDescription() != null) { - builder.setDescription(input.getDescription()); - } - if (input.getImmutable() != null) { - builder.setImmutable(input.getImmutable()); - } - if (input.getTypeQualifier() != null) { - buildTypeQualifier(input, builder, existingDefinition); - } - if (input.getNewAllowedValues() != null) { - buildAllowedValues(input, builder); - } - if (input.getSetCardinalityAsMultiple() != null) { - builder.setCardinality(PropertyCardinality.MULTIPLE); + List mcps = new ArrayList<>(); + + // first update the definition aspect if we need to + MetadataChangeProposal definitionMcp = + updateDefinition(input, context, propertyUrn, entityResponse); + if (definitionMcp != null) { + mcps.add(definitionMcp); } - if (input.getNewEntityTypes() != null) { - input.getNewEntityTypes().forEach(builder::addEntityType); + + // then update the settings aspect if we need to + if (input.getSettings() != null) { + mcps.add(updateSettings(context, input.getSettings(), propertyUrn, entityResponse)); } - builder.setLastModified(context.getOperationContext().getAuditStamp()); - MetadataChangeProposal mcp = builder.build(); - _entityClient.ingestProposal(context.getOperationContext(), mcp, false); + _entityClient.batchIngestProposals(context.getOperationContext(), mcps, false); EntityResponse response = _entityClient.getV2( @@ -102,6 +95,120 @@ public CompletableFuture get(final DataFetchingEnviron }); } + private boolean hasSettingsChanged( + StructuredPropertySettings existingSettings, StructuredPropertySettingsInput settingsInput) { + if (settingsInput.getIsHidden() != null + && !existingSettings.isIsHidden().equals(settingsInput.getIsHidden())) { + return true; + } + if (settingsInput.getShowInSearchFilters() != null + && !existingSettings + .isShowInSearchFilters() + .equals(settingsInput.getShowInSearchFilters())) { + return true; + } + if (settingsInput.getShowInAssetSummary() != null + && !existingSettings.isShowInAssetSummary().equals(settingsInput.getShowInAssetSummary())) { + return true; + } + if (settingsInput.getShowAsAssetBadge() != null + && !existingSettings.isShowAsAssetBadge().equals(settingsInput.getShowAsAssetBadge())) { + return true; + } + if (settingsInput.getShowInColumnsTable() != null + && !existingSettings.isShowInColumnsTable().equals(settingsInput.getShowInColumnsTable())) { + return true; + } + return false; + } + + private MetadataChangeProposal updateSettings( + @Nonnull final QueryContext context, + @Nonnull final StructuredPropertySettingsInput settingsInput, + @Nonnull final Urn propertyUrn, + @Nonnull final EntityResponse entityResponse) + throws Exception { + StructuredPropertySettings existingSettings = + getExistingStructuredPropertySettings(entityResponse); + // check if settings has changed to determine if we should update the timestamp + boolean hasChanged = hasSettingsChanged(existingSettings, settingsInput); + if (hasChanged) { + existingSettings.setLastModified(context.getOperationContext().getAuditStamp()); + } + + if (settingsInput.getIsHidden() != null) { + existingSettings.setIsHidden(settingsInput.getIsHidden()); + } + if (settingsInput.getShowInSearchFilters() != null) { + existingSettings.setShowInSearchFilters(settingsInput.getShowInSearchFilters()); + } + if (settingsInput.getShowInAssetSummary() != null) { + existingSettings.setShowInAssetSummary(settingsInput.getShowInAssetSummary()); + } + if (settingsInput.getShowAsAssetBadge() != null) { + existingSettings.setShowAsAssetBadge(settingsInput.getShowAsAssetBadge()); + } + if (settingsInput.getShowInColumnsTable() != null) { + existingSettings.setShowInColumnsTable(settingsInput.getShowInColumnsTable()); + } + + StructuredPropertyUtils.validatePropertySettings(existingSettings, true); + + return buildMetadataChangeProposalWithUrn( + propertyUrn, STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME, existingSettings); + } + + private MetadataChangeProposal updateDefinition( + @Nonnull final UpdateStructuredPropertyInput input, + @Nonnull final QueryContext context, + @Nonnull final Urn propertyUrn, + @Nonnull final EntityResponse entityResponse) + throws Exception { + StructuredPropertyDefinition existingDefinition = + getExistingStructuredPropertyDefinition(entityResponse); + StructuredPropertyDefinitionPatchBuilder builder = + new StructuredPropertyDefinitionPatchBuilder().urn(propertyUrn); + + boolean hasUpdatedDefinition = false; + + if (input.getDisplayName() != null) { + builder.setDisplayName(input.getDisplayName()); + hasUpdatedDefinition = true; + } + if (input.getDescription() != null) { + builder.setDescription(input.getDescription()); + hasUpdatedDefinition = true; + } + if (input.getImmutable() != null) { + builder.setImmutable(input.getImmutable()); + hasUpdatedDefinition = true; + } + if (input.getTypeQualifier() != null) { + buildTypeQualifier(input, builder, existingDefinition); + hasUpdatedDefinition = true; + } + if (input.getNewAllowedValues() != null) { + buildAllowedValues(input, builder); + hasUpdatedDefinition = true; + } + if (input.getSetCardinalityAsMultiple() != null + && input.getSetCardinalityAsMultiple().equals(true)) { + builder.setCardinality(PropertyCardinality.MULTIPLE); + hasUpdatedDefinition = true; + } + if (input.getNewEntityTypes() != null) { + input.getNewEntityTypes().forEach(builder::addEntityType); + hasUpdatedDefinition = true; + } + + if (hasUpdatedDefinition) { + builder.setLastModified(context.getOperationContext().getAuditStamp()); + + return builder.build(); + } + return null; + } + private void buildTypeQualifier( @Nonnull final UpdateStructuredPropertyInput input, @Nonnull final StructuredPropertyDefinitionPatchBuilder builder, @@ -141,17 +248,40 @@ private void buildAllowedValues( }); } - private StructuredPropertyDefinition getExistingStructuredProperty( + private EntityResponse getExistingStructuredProperty( @Nonnull final QueryContext context, @Nonnull final Urn propertyUrn) throws Exception { - EntityResponse response = - _entityClient.getV2( - context.getOperationContext(), STRUCTURED_PROPERTY_ENTITY_NAME, propertyUrn, null); + return _entityClient.getV2( + context.getOperationContext(), STRUCTURED_PROPERTY_ENTITY_NAME, propertyUrn, null); + } + private StructuredPropertyDefinition getExistingStructuredPropertyDefinition( + EntityResponse response) throws Exception { if (response != null && response.getAspects().containsKey(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME)) { return new StructuredPropertyDefinition( - response.getAspects().get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME).getValue().data()); + response + .getAspects() + .get(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME) + .getValue() + .data() + .copy()); } - return null; + throw new IllegalArgumentException( + "Attempting to update a structured property with no definition aspect."); + } + + private StructuredPropertySettings getExistingStructuredPropertySettings(EntityResponse response) + throws Exception { + if (response != null + && response.getAspects().containsKey(STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME)) { + return new StructuredPropertySettings( + response + .getAspects() + .get(STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME) + .getValue() + .data() + .copy()); + } + return new StructuredPropertySettings(); } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java index 770c8a0d749c38..6c1d7949332fbe 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpsertStructuredPropertiesResolver.java @@ -103,7 +103,7 @@ public CompletableFuture chart.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java index 02357b3ddc349e..7ac00c46475bce 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/container/mappers/ContainerMapper.java @@ -161,7 +161,9 @@ public static Container map( if (envelopedStructuredProps != null) { result.setStructuredProperties( StructuredPropertiesMapper.map( - context, new StructuredProperties(envelopedStructuredProps.getValue().data()))); + context, + new StructuredProperties(envelopedStructuredProps.getValue().data()), + entityUrn)); } final EnvelopedAspect envelopedForms = aspects.get(FORMS_ASPECT_NAME); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java index 6246cf64bbf7f8..010816431f54de 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpgroup/mappers/CorpGroupMapper.java @@ -59,7 +59,8 @@ public CorpGroup apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java index 4fa278983399b1..a94b555daebdfb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java @@ -88,7 +88,8 @@ public CorpUser apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java index 4fa52b11365641..fd1c7a5db2a79d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dashboard/mappers/DashboardMapper.java @@ -142,7 +142,8 @@ public Dashboard apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((dashboard, dataMap) -> dashboard.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java index 9e2612f60abda1..44bc6a99eae4bb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataflow/mappers/DataFlowMapper.java @@ -114,7 +114,8 @@ public DataFlow apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java index d7da875bc2a29f..772871d77f2175 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/datajob/mappers/DataJobMapper.java @@ -135,7 +135,8 @@ public DataJob apply( result.setSubTypes(SubTypesMapper.map(context, new SubTypes(data))); } else if (STRUCTURED_PROPERTIES_ASPECT_NAME.equals(name)) { result.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(data))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(data), entityUrn)); } else if (FORMS_ASPECT_NAME.equals(name)) { result.setForms(FormsMapper.map(new Forms(data), entityUrn.toString())); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java index 08637dbfd01edc..8693ec97f1a2ee 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataproduct/mappers/DataProductMapper.java @@ -92,7 +92,8 @@ public DataProduct apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java index 0869463ba73ac2..e411014c23c89b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/dataset/mappers/DatasetMapper.java @@ -173,7 +173,8 @@ public Dataset apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((dataset, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java index 7d05e0862a96da..ffcb94a0b7e29e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/domain/DomainMapper.java @@ -71,7 +71,9 @@ public static Domain map(@Nullable QueryContext context, final EntityResponse en if (envelopedStructuredProps != null) { result.setStructuredProperties( StructuredPropertiesMapper.map( - context, new StructuredProperties(envelopedStructuredProps.getValue().data()))); + context, + new StructuredProperties(envelopedStructuredProps.getValue().data()), + entityUrn)); } final EnvelopedAspect envelopedForms = aspects.get(FORMS_ASPECT_NAME); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java index 4912d18614f415..a694b62999080e 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryNodeMapper.java @@ -59,7 +59,8 @@ public GlossaryNode apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java index 1274646f45ec49..e309ffad84df58 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/glossary/mappers/GlossaryTermMapper.java @@ -90,7 +90,8 @@ public GlossaryTerm apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java index a4f3aa7a0e2261..d5eb1a15624dc3 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureMapper.java @@ -115,7 +115,8 @@ public MLFeature apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((mlFeature, dataMap) -> mlFeature.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java index 30bf4dda1cf4fd..51d3004d97a619 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLFeatureTableMapper.java @@ -117,7 +117,8 @@ public MLFeatureTable apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((mlFeatureTable, dataMap) -> mlFeatureTable.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java index 7e99040e44c82e..6e3da1c1533926 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelGroupMapper.java @@ -112,7 +112,8 @@ public MLModelGroup apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((mlModelGroup, dataMap) -> mlModelGroup.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java index a3bc5c663c89ae..7102fd4aed9743 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLModelMapper.java @@ -174,7 +174,8 @@ public MLModel apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((dataset, dataMap) -> dataset.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java index 36784f96ea30ea..c446c892cb2231 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mlmodel/mappers/MLPrimaryKeyMapper.java @@ -112,7 +112,8 @@ public MLPrimaryKey apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((entity, dataMap) -> entity.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( FORMS_ASPECT_NAME, ((entity, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java index b1f27357d45504..30eac54aede9bb 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/schemafield/SchemaFieldMapper.java @@ -41,7 +41,8 @@ public SchemaFieldEntity apply( STRUCTURED_PROPERTIES_ASPECT_NAME, ((schemaField, dataMap) -> schemaField.setStructuredProperties( - StructuredPropertiesMapper.map(context, new StructuredProperties(dataMap))))); + StructuredPropertiesMapper.map( + context, new StructuredProperties(dataMap), entityUrn)))); mappingHelper.mapToResult( BUSINESS_ATTRIBUTE_ASPECT, (((schemaField, dataMap) -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java index dc1ff7ca329714..4f155903c055b1 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertiesMapper.java @@ -25,23 +25,29 @@ public class StructuredPropertiesMapper { public static final StructuredPropertiesMapper INSTANCE = new StructuredPropertiesMapper(); public static com.linkedin.datahub.graphql.generated.StructuredProperties map( - @Nullable QueryContext context, @Nonnull final StructuredProperties structuredProperties) { - return INSTANCE.apply(context, structuredProperties); + @Nullable QueryContext context, + @Nonnull final StructuredProperties structuredProperties, + @Nonnull final Urn entityUrn) { + return INSTANCE.apply(context, structuredProperties, entityUrn); } public com.linkedin.datahub.graphql.generated.StructuredProperties apply( - @Nullable QueryContext context, @Nonnull final StructuredProperties structuredProperties) { + @Nullable QueryContext context, + @Nonnull final StructuredProperties structuredProperties, + @Nonnull final Urn entityUrn) { com.linkedin.datahub.graphql.generated.StructuredProperties result = new com.linkedin.datahub.graphql.generated.StructuredProperties(); result.setProperties( structuredProperties.getProperties().stream() - .map(p -> mapStructuredProperty(context, p)) + .map(p -> mapStructuredProperty(context, p, entityUrn)) .collect(Collectors.toList())); return result; } private StructuredPropertiesEntry mapStructuredProperty( - @Nullable QueryContext context, StructuredPropertyValueAssignment valueAssignment) { + @Nullable QueryContext context, + StructuredPropertyValueAssignment valueAssignment, + @Nonnull final Urn entityUrn) { StructuredPropertiesEntry entry = new StructuredPropertiesEntry(); entry.setStructuredProperty(createStructuredPropertyEntity(valueAssignment)); final List values = new ArrayList<>(); @@ -58,6 +64,7 @@ private StructuredPropertiesEntry mapStructuredProperty( }); entry.setValues(values); entry.setValueEntities(entities); + entry.setAssociatedUrn(entityUrn.toString()); return entry; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java index c539c65118ac6d..5dc73d9ad09388 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyMapper.java @@ -17,6 +17,7 @@ import com.linkedin.datahub.graphql.generated.StringValue; import com.linkedin.datahub.graphql.generated.StructuredPropertyDefinition; import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.StructuredPropertySettings; import com.linkedin.datahub.graphql.generated.TypeQualifier; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; import com.linkedin.datahub.graphql.types.mappers.MapperUtils; @@ -55,6 +56,8 @@ public StructuredPropertyEntity apply( MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); mappingHelper.mapToResult( STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, (this::mapStructuredPropertyDefinition)); + mappingHelper.mapToResult( + STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME, (this::mapStructuredPropertySettings)); return mappingHelper.getResult(); } @@ -112,6 +115,21 @@ private List mapAllowedValues(@Nonnull PropertyValueArray gmsValue return allowedValues; } + private void mapStructuredPropertySettings( + @Nonnull StructuredPropertyEntity extendedProperty, @Nonnull DataMap dataMap) { + com.linkedin.structured.StructuredPropertySettings gmsSettings = + new com.linkedin.structured.StructuredPropertySettings(dataMap); + StructuredPropertySettings settings = new StructuredPropertySettings(); + + settings.setIsHidden(gmsSettings.isIsHidden()); + settings.setShowInSearchFilters(gmsSettings.isShowInSearchFilters()); + settings.setShowInAssetSummary(gmsSettings.isShowInAssetSummary()); + settings.setShowAsAssetBadge(gmsSettings.isShowAsAssetBadge()); + settings.setShowInColumnsTable(gmsSettings.isShowInColumnsTable()); + + extendedProperty.setSettings(settings); + } + private DataTypeEntity createDataTypeEntity(final Urn dataTypeUrn) { final DataTypeEntity dataType = new DataTypeEntity(); dataType.setUrn(dataTypeUrn.toString()); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java index 22e161d320f215..e451e96a3e84d9 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/structuredproperty/StructuredPropertyType.java @@ -27,7 +27,8 @@ public class StructuredPropertyType implements com.linkedin.datahub.graphql.types.EntityType { public static final Set ASPECTS_TO_FETCH = - ImmutableSet.of(STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME); + ImmutableSet.of( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME); private final EntityClient _entityClient; @Override diff --git a/datahub-graphql-core/src/main/resources/properties.graphql b/datahub-graphql-core/src/main/resources/properties.graphql index 292381d064f362..ff20caa50bf036 100644 --- a/datahub-graphql-core/src/main/resources/properties.graphql +++ b/datahub-graphql-core/src/main/resources/properties.graphql @@ -49,6 +49,11 @@ type StructuredPropertyEntity implements Entity { """ definition: StructuredPropertyDefinition! + """ + Definition of this structured property including its name + """ + settings: StructuredPropertySettings + """ Granular API for querying edges extending from this entity """ @@ -117,6 +122,36 @@ type StructuredPropertyDefinition { lastModified: ResolvedAuditStamp } +""" +Settings specific to a structured property entity +""" +type StructuredPropertySettings { + """ + Whether or not this asset should be hidden in the main application + """ + isHidden: Boolean! + + """ + Whether or not this asset should be displayed as a search filter + """ + showInSearchFilters: Boolean! + + """ + Whether or not this asset should be displayed in the asset sidebar + """ + showInAssetSummary: Boolean! + + """ + Whether or not this asset should be displayed as an asset badge on other asset's headers + """ + showAsAssetBadge: Boolean! + + """ + Whether or not this asset should be displayed as a column in the schema field table in a Dataset's "Columns" tab. + """ + showInColumnsTable: Boolean! +} + """ An entry for an allowed value for a structured property """ @@ -202,6 +237,11 @@ type StructuredPropertiesEntry { The optional entities associated with the values if the values are entity urns """ valueEntities: [Entity] + + """ + The urn of the entity this property came from for tracking purposes e.g. when sibling nodes are merged together + """ + associatedUrn: String! } """ @@ -330,8 +370,9 @@ input CreateStructuredPropertyInput { """ The unique fully qualified name of this structured property, dot delimited. + This will be required to match the ID of this structured property. """ - qualifiedName: String! + qualifiedName: String """ The optional display name for this property @@ -375,6 +416,11 @@ input CreateStructuredPropertyInput { For example: ["urn:li:entityType:datahub.dataset"] """ entityTypes: [String!]! + + """ + Settings for this structured property + """ + settings: StructuredPropertySettingsInput } """ @@ -455,6 +501,11 @@ input UpdateStructuredPropertyInput { For backwards compatibility, this is append only. """ newEntityTypes: [String!] + + """ + Settings for this structured property + """ + settings: StructuredPropertySettingsInput } """ @@ -477,3 +528,34 @@ input DeleteStructuredPropertyInput { """ urn: String! } + +""" +Settings for a structured property +""" +input StructuredPropertySettingsInput { + """ + Whether or not this asset should be hidden in the main application + """ + isHidden: Boolean + + """ + Whether or not this asset should be displayed as a search filter + """ + showInSearchFilters: Boolean + + """ + Whether or not this asset should be displayed in the asset sidebar + """ + showInAssetSummary: Boolean + + """ + Whether or not this asset should be displayed as an asset badge on other asset's headers + """ + showAsAssetBadge: Boolean + + """ + Whether or not this asset should be displayed as a column in the schema field table in a Dataset's "Columns" tab. + """ + showInColumnsTable: Boolean +} + diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java index 72cdb78542e414..fec2251f92b63f 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/CreateStructuredPropertyResolverTest.java @@ -10,11 +10,11 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.CreateStructuredPropertyInput; import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.StructuredPropertySettingsInput; import com.linkedin.entity.EntityResponse; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; -import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; import graphql.schema.DataFetchingEnvironment; import java.util.ArrayList; @@ -36,7 +36,8 @@ public class CreateStructuredPropertyResolverTest { null, null, null, - new ArrayList<>()); + new ArrayList<>(), + null); @Test public void testGetSuccess() throws Exception { @@ -56,7 +57,40 @@ public void testGetSuccess() throws Exception { // Validate that we called ingest Mockito.verify(mockEntityClient, Mockito.times(1)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testGetMismatchIdAndQualifiedName() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + CreateStructuredPropertyInput testInput = + new CreateStructuredPropertyInput( + "mismatched", + "io.acryl.test", + "Display Name", + "description", + true, + null, + null, + null, + null, + new ArrayList<>(), + null); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate ingest is not called + Mockito.verify(mockEntityClient, Mockito.times(0)) + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } @Test @@ -75,7 +109,7 @@ public void testGetUnauthorized() throws Exception { // Validate that we did NOT call ingest Mockito.verify(mockEntityClient, Mockito.times(0)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } @Test @@ -94,7 +128,83 @@ public void testGetFailure() throws Exception { // Validate that ingest was called, but that caused a failure Mockito.verify(mockEntityClient, Mockito.times(1)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testGetInvalidSettingsInput() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + // if isHidden is true, other fields should not be true + StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput(); + settingsInput.setIsHidden(true); + settingsInput.setShowAsAssetBadge(true); + + CreateStructuredPropertyInput testInput = + new CreateStructuredPropertyInput( + null, + "io.acryl.test", + "Display Name", + "description", + true, + null, + null, + null, + null, + new ArrayList<>(), + settingsInput); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate ingest is not called + Mockito.verify(mockEntityClient, Mockito.times(0)) + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testGetSuccessWithSettings() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + CreateStructuredPropertyResolver resolver = + new CreateStructuredPropertyResolver(mockEntityClient); + + StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput(); + settingsInput.setShowAsAssetBadge(true); + + CreateStructuredPropertyInput testInput = + new CreateStructuredPropertyInput( + null, + "io.acryl.test", + "Display Name", + "description", + true, + null, + null, + null, + null, + new ArrayList<>(), + settingsInput); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + StructuredPropertyEntity prop = resolver.get(mockEnv).get(); + + assertEquals(prop.getUrn(), TEST_STRUCTURED_PROPERTY_URN); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolverTest.java new file mode 100644 index 00000000000000..7ecec25708f2d5 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/DeleteStructuredPropertyResolverTest.java @@ -0,0 +1,91 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; +import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static org.mockito.ArgumentMatchers.any; +import static org.testng.Assert.assertThrows; +import static org.testng.Assert.assertTrue; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DeleteStructuredPropertyInput; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +public class DeleteStructuredPropertyResolverTest { + private static final String TEST_PROP_URN = "urn:li:structuredProperty:test"; + + private static final DeleteStructuredPropertyInput TEST_INPUT = + new DeleteStructuredPropertyInput(TEST_PROP_URN); + + @Test + public void testGetSuccess() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + DeleteStructuredPropertyResolver resolver = + new DeleteStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Boolean success = resolver.get(mockEnv).get(); + assertTrue(success); + + // Validate that we called delete + Mockito.verify(mockEntityClient, Mockito.times(1)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_PROP_URN))); + } + + @Test + public void testGetUnauthorized() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + DeleteStructuredPropertyResolver resolver = + new DeleteStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockDenyContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that we did NOT call delete + Mockito.verify(mockEntityClient, Mockito.times(0)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_PROP_URN))); + } + + @Test + public void testGetFailure() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(false); + DeleteStructuredPropertyResolver resolver = + new DeleteStructuredPropertyResolver(mockEntityClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that deleteEntity was called, but since it's the thing that failed it was called + // once still + Mockito.verify(mockEntityClient, Mockito.times(1)) + .deleteEntity(any(), Mockito.eq(UrnUtils.getUrn(TEST_PROP_URN))); + } + + private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + if (!shouldSucceed) { + Mockito.doThrow(new RemoteInvocationException()).when(client).deleteEntity(any(), any()); + } + return client; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/StructuredPropertyUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/StructuredPropertyUtilsTest.java new file mode 100644 index 00000000000000..0e9d064b3c7af7 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/StructuredPropertyUtilsTest.java @@ -0,0 +1,42 @@ +package com.linkedin.datahub.graphql.resolvers.structuredproperties; + +import static org.testng.Assert.*; + +import com.linkedin.metadata.models.StructuredPropertyUtils; +import java.util.UUID; +import org.testng.annotations.Test; + +public class StructuredPropertyUtilsTest { + + @Test + public void testGetIdMismatchedInput() throws Exception { + assertThrows( + IllegalArgumentException.class, + () -> StructuredPropertyUtils.getPropertyId("test1", "test2")); + } + + @Test + public void testGetIdConsistentInput() throws Exception { + assertEquals(StructuredPropertyUtils.getPropertyId("test1", "test1"), "test1"); + } + + @Test + public void testGetIdNullQualifiedName() throws Exception { + assertEquals(StructuredPropertyUtils.getPropertyId("test1", null), "test1"); + } + + @Test + public void testGetIdNullId() throws Exception { + assertEquals(StructuredPropertyUtils.getPropertyId(null, "test1"), "test1"); + } + + @Test + public void testGetIdNullForBoth() throws Exception { + try { + String id = StructuredPropertyUtils.getPropertyId(null, null); + UUID.fromString(id); + } catch (Exception e) { + fail("ID produced is not a UUID"); + } + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java index b818bcfb7d7f4f..2b0e7fd83b7cee 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/structuredproperties/UpdateStructuredPropertyResolverTest.java @@ -2,20 +2,25 @@ import static com.linkedin.datahub.graphql.TestUtils.getMockAllowContext; import static com.linkedin.datahub.graphql.TestUtils.getMockDenyContext; +import static com.linkedin.metadata.Constants.STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME; import static org.mockito.ArgumentMatchers.any; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertThrows; +import com.linkedin.common.UrnArray; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.StructuredPropertyEntity; +import com.linkedin.datahub.graphql.generated.StructuredPropertySettingsInput; import com.linkedin.datahub.graphql.generated.UpdateStructuredPropertyInput; +import com.linkedin.entity.Aspect; import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; import com.linkedin.entity.EnvelopedAspectMap; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.Constants; -import com.linkedin.mxe.MetadataChangeProposal; import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.structured.StructuredPropertyDefinition; import graphql.schema.DataFetchingEnvironment; import java.util.concurrent.CompletionException; import org.mockito.Mockito; @@ -33,6 +38,7 @@ public class UpdateStructuredPropertyResolverTest { null, null, null, + null, null); @Test @@ -53,7 +59,7 @@ public void testGetSuccess() throws Exception { // Validate that we called ingest Mockito.verify(mockEntityClient, Mockito.times(1)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } @Test @@ -72,7 +78,7 @@ public void testGetUnauthorized() throws Exception { // Validate that we did NOT call ingest Mockito.verify(mockEntityClient, Mockito.times(0)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } @Test @@ -91,7 +97,80 @@ public void testGetFailure() throws Exception { // Validate that ingest was not called since there was a get failure before ingesting Mockito.verify(mockEntityClient, Mockito.times(0)) - .ingestProposal(any(), any(MetadataChangeProposal.class), Mockito.eq(false)); + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testGetInvalidSettingsInput() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateStructuredPropertyResolver resolver = + new UpdateStructuredPropertyResolver(mockEntityClient); + + // if isHidden is true, other fields should not be true + StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput(); + settingsInput.setIsHidden(true); + settingsInput.setShowInSearchFilters(true); + + final UpdateStructuredPropertyInput testInput = + new UpdateStructuredPropertyInput( + TEST_STRUCTURED_PROPERTY_URN, + "New Display Name", + "new description", + true, + null, + null, + null, + null, + settingsInput); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + + // Validate that ingest was not called since there was a get failure before ingesting + Mockito.verify(mockEntityClient, Mockito.times(0)) + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); + } + + @Test + public void testGetValidSettingsInput() throws Exception { + EntityClient mockEntityClient = initMockEntityClient(true); + UpdateStructuredPropertyResolver resolver = + new UpdateStructuredPropertyResolver(mockEntityClient); + + // if isHidden is true, other fields should not be true + StructuredPropertySettingsInput settingsInput = new StructuredPropertySettingsInput(); + settingsInput.setIsHidden(true); + + final UpdateStructuredPropertyInput testInput = + new UpdateStructuredPropertyInput( + TEST_STRUCTURED_PROPERTY_URN, + "New Display Name", + "new description", + true, + null, + null, + null, + null, + settingsInput); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + StructuredPropertyEntity prop = resolver.get(mockEnv).get(); + + assertEquals(prop.getUrn(), TEST_STRUCTURED_PROPERTY_URN); + + // Validate that we called ingest + Mockito.verify(mockEntityClient, Mockito.times(1)) + .batchIngestProposals(any(), Mockito.anyList(), Mockito.eq(false)); } private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exception { @@ -99,7 +178,11 @@ private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exceptio EntityResponse response = new EntityResponse(); response.setEntityName(Constants.STRUCTURED_PROPERTY_ENTITY_NAME); response.setUrn(UrnUtils.getUrn(TEST_STRUCTURED_PROPERTY_URN)); - response.setAspects(new EnvelopedAspectMap()); + final EnvelopedAspectMap aspectMap = new EnvelopedAspectMap(); + aspectMap.put( + STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(createDefinition().data()))); + response.setAspects(aspectMap); if (shouldSucceed) { Mockito.when( client.getV2( @@ -120,4 +203,13 @@ private EntityClient initMockEntityClient(boolean shouldSucceed) throws Exceptio return client; } + + private StructuredPropertyDefinition createDefinition() { + StructuredPropertyDefinition definition = new StructuredPropertyDefinition(); + definition.setDisplayName("test"); + definition.setQualifiedName("test"); + definition.setValueType(UrnUtils.getUrn("urn:li:dataType:datahub.string")); + definition.setEntityTypes(new UrnArray()); + return definition; + } } diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java index e9ee7789550c6c..1b12f540badfb8 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/StructuredPropertyUtils.java @@ -20,6 +20,7 @@ import com.linkedin.structured.PrimitivePropertyValue; import com.linkedin.structured.StructuredProperties; import com.linkedin.structured.StructuredPropertyDefinition; +import com.linkedin.structured.StructuredPropertySettings; import com.linkedin.structured.StructuredPropertyValueAssignment; import com.linkedin.structured.StructuredPropertyValueAssignmentArray; import com.linkedin.util.Pair; @@ -32,6 +33,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.stream.Collectors; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -45,6 +47,11 @@ private StructuredPropertyUtils() {} static final Date MIN_DATE = Date.valueOf("1000-01-01"); static final Date MAX_DATE = Date.valueOf("9999-12-31"); + public static final String INVALID_SETTINGS_MESSAGE = + "Cannot have property isHidden = true while other display location settings are also true."; + public static final String ONLY_ONE_BADGE = + "Cannot have more than one property set with show as badge. Property urns currently set: "; + public static LogicalValueType getLogicalValueType( StructuredPropertyDefinition structuredPropertyDefinition) { return getLogicalValueType(structuredPropertyDefinition.getValueType()); @@ -355,4 +362,47 @@ private static Pair filterValue true); } } + + /* + * We accept both ID and qualifiedName as inputs when creating a structured property. However, + * these two fields should ALWAYS be the same. If they don't provide either, use a UUID for both. + * If they provide both, ensure they are the same otherwise throw. Otherwise, use what is provided. + */ + public static String getPropertyId( + @Nullable final String inputId, @Nullable final String inputQualifiedName) { + if (inputId != null && inputQualifiedName != null && !inputId.equals(inputQualifiedName)) { + throw new IllegalArgumentException( + "Qualified name and the ID of a structured property must match"); + } + + String id = UUID.randomUUID().toString(); + + if (inputQualifiedName != null) { + id = inputQualifiedName; + } else if (inputId != null) { + id = inputId; + } + + return id; + } + + /* + * Ensure that a structured property settings aspect is valid by ensuring that if isHidden is true, + * the other fields concerning display locations are false; + */ + public static boolean validatePropertySettings( + StructuredPropertySettings settings, boolean shouldThrow) { + if (settings.isIsHidden()) { + if (settings.isShowInSearchFilters() + || settings.isShowInAssetSummary() + || settings.isShowAsAssetBadge()) { + if (shouldThrow) { + throw new IllegalArgumentException(INVALID_SETTINGS_MESSAGE); + } else { + return false; + } + } + } + return true; + } } diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 9c608187342e8c..797055d5fb6a93 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -363,6 +363,8 @@ public class Constants { // Structured Property public static final String STRUCTURED_PROPERTY_DEFINITION_ASPECT_NAME = "propertyDefinition"; public static final String STRUCTURED_PROPERTY_KEY_ASPECT_NAME = "structuredPropertyKey"; + public static final String STRUCTURED_PROPERTY_SETTINGS_ASPECT_NAME = + "structuredPropertySettings"; // Form public static final String FORM_INFO_ASPECT_NAME = "formInfo"; diff --git a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py index 181c70adc640a6..013efbdf6a2f6b 100644 --- a/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py +++ b/metadata-ingestion/src/datahub/api/entities/structuredproperties/structuredproperties.py @@ -118,11 +118,13 @@ def validate_entity_types(cls, v): @property def fqn(self) -> str: assert self.urn is not None - return ( - self.qualified_name - or self.id - or Urn.from_string(self.urn).get_entity_id()[0] - ) + id = Urn.create_from_string(self.urn).get_entity_id()[0] + if self.qualified_name is not None: + # ensure that qualified name and ID match + assert ( + self.qualified_name == id + ), "ID in the urn and the qualified_name must match" + return id @validator("urn", pre=True, always=True) def urn_must_be_present(cls, v, values): diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl index 3ddb2d2e571da3..416e2c5c11e228 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertyDefinition.pdl @@ -89,25 +89,25 @@ record StructuredPropertyDefinition { version: optional string /** - * Created Audit stamp - */ - @Searchable = { - "/time": { - "fieldName": "createdTime", - "fieldType": "DATETIME" - } - } - created: optional AuditStamp + * Created Audit stamp + */ + @Searchable = { + "/time": { + "fieldName": "createdTime", + "fieldType": "DATETIME" + } + } + created: optional AuditStamp - /** - * Created Audit stamp - */ - @Searchable = { - "/time": { - "fieldName": "lastModified", - "fieldType": "DATETIME" - } - } - lastModified: optional AuditStamp + /** + * Last Modified Audit stamp + */ + @Searchable = { + "/time": { + "fieldName": "lastModified", + "fieldType": "DATETIME" + } + } + lastModified: optional AuditStamp } diff --git a/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertySettings.pdl b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertySettings.pdl new file mode 100644 index 00000000000000..fadcdfa5204e14 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/structured/StructuredPropertySettings.pdl @@ -0,0 +1,64 @@ +namespace com.linkedin.structured + +import com.linkedin.common.AuditStamp + +/** + * Settings specific to a structured property entity + */ +@Aspect = { + "name": "structuredPropertySettings" +} +record StructuredPropertySettings { + /** + * Whether or not this asset should be hidden in the main application + */ + @Searchable = { + "fieldType": "BOOLEAN" + } + isHidden: boolean = false + + /** + * Whether or not this asset should be displayed as a search filter + */ + @Searchable = { + "fieldType": "BOOLEAN" + } + showInSearchFilters: boolean = false + + /** + * Whether or not this asset should be displayed in the asset sidebar + */ + @Searchable = { + "fieldType": "BOOLEAN" + } + showInAssetSummary: boolean = false + + /** + * Whether or not this asset should be displayed as an asset badge on other + * asset's headers + */ + @Searchable = { + "fieldType": "BOOLEAN" + } + showAsAssetBadge: boolean = false + + /** + * Whether or not this asset should be displayed as a column in the schema field table + * in a Dataset's "Columns" tab. + */ + @Searchable = { + "fieldType": "BOOLEAN" + } + showInColumnsTable: boolean = false + + /** + * Last Modified Audit stamp + */ + @Searchable = { + "/time": { + "fieldName": "lastModifiedSettings", + "fieldType": "DATETIME" + } + } + lastModified: optional AuditStamp +} diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index ee1481f29f7e9f..1c3eb5b574e204 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -602,6 +602,7 @@ entities: keyAspect: structuredPropertyKey aspects: - propertyDefinition + - structuredPropertySettings - institutionalMemory - status - name: form From d062411c6893d2d34c4cf14b48eb6f5acb7d1658 Mon Sep 17 00:00:00 2001 From: sid-acryl <155424659+sid-acryl@users.noreply.github.com> Date: Thu, 12 Dec 2024 01:08:52 +0530 Subject: [PATCH 03/16] fix(ingest/tableau): project_path_pattern use in _is_denied_project (#12010) --- .../src/datahub/configuration/common.py | 6 +-- .../ingestion/source/tableau/tableau.py | 51 +++++++++++++------ 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py index 4fdf564162410c..7df007e087979c 100644 --- a/metadata-ingestion/src/datahub/configuration/common.py +++ b/metadata-ingestion/src/datahub/configuration/common.py @@ -258,7 +258,7 @@ def allow_all(cls) -> "AllowDenyPattern": return AllowDenyPattern() def allowed(self, string: str) -> bool: - if self._denied(string): + if self.denied(string): return False return any( @@ -266,7 +266,7 @@ def allowed(self, string: str) -> bool: for allow_pattern in self.allow ) - def _denied(self, string: str) -> bool: + def denied(self, string: str) -> bool: for deny_pattern in self.deny: if re.match(deny_pattern, string, self.regex_flags): return True @@ -290,7 +290,7 @@ def get_allowed_list(self) -> List[str]: raise ValueError( "allow list must be fully specified to get list of allowed strings" ) - return [a for a in self.allow if not self._denied(a)] + return [a for a in self.allow if not self.denied(a)] def __eq__(self, other): # type: ignore return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ diff --git a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py index 68c38d4d064612..6844b8a425a7b6 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py +++ b/metadata-ingestion/src/datahub/ingestion/source/tableau/tableau.py @@ -353,7 +353,7 @@ class TableauConfig( project_path_separator: str = Field( default="/", - description="The separator used for the project_pattern field between project names. By default, we use a slash. " + description="The separator used for the project_path_pattern field between project names. By default, we use a slash. " "You can change this if your Tableau projects contain slashes in their names, and you'd like to filter by project.", ) @@ -959,19 +959,36 @@ def _is_allowed_project(self, project: TableauProject) -> bool: return is_allowed def _is_denied_project(self, project: TableauProject) -> bool: - # Either project name or project path should exist in deny - for deny_pattern in self.config.project_pattern.deny: - # Either name or project path is denied - if re.match( - deny_pattern, project.name, self.config.project_pattern.regex_flags - ) or re.match( - deny_pattern, - self._get_project_path(project), - self.config.project_pattern.regex_flags, - ): - return True - logger.info(f"project({project.name}) is not denied as per project_pattern") - return False + """ + Why use an explicit denial check instead of the `AllowDenyPattern.allowed` method? + + Consider a scenario where a Tableau site contains four projects: A, B, C, and D, with the following hierarchical relationship: + + - **A** + - **B** (Child of A) + - **C** (Child of A) + - **D** + + In this setup: + + - `project_pattern` is configured with `allow: ["A"]` and `deny: ["B"]`. + - `extract_project_hierarchy` is set to `True`. + + The goal is to extract assets from project A and its children while explicitly denying the child project B. + + If we rely solely on the `project_pattern.allowed()` method, project C's assets will not be ingested. + This happens because project C is not explicitly included in the `allow` list, nor is it part of the `deny` list. + However, since `extract_project_hierarchy` is enabled, project C should ideally be included in the ingestion process unless explicitly denied. + + To address this, the function explicitly checks the deny regex to ensure that project C’s assets are ingested if it is not specifically denied in the deny list. This approach ensures that the hierarchy is respected while adhering to the configured allow/deny rules. + """ + + # Either project_pattern or project_path_pattern is set in a recipe + # TableauConfig.projects_backward_compatibility ensures that at least one of these properties is configured. + + return self.config.project_pattern.denied( + project.name + ) or self.config.project_path_pattern.denied(self._get_project_path(project)) def _init_tableau_project_registry(self, all_project_map: dict) -> None: list_of_skip_projects: List[TableauProject] = [] @@ -999,9 +1016,11 @@ def _init_tableau_project_registry(self, all_project_map: dict) -> None: for project in list_of_skip_projects: if ( project.parent_id in projects_to_ingest - and self._is_denied_project(project) is False + and not self._is_denied_project(project) ): - logger.debug(f"Project {project.name} is added in project registry") + logger.debug( + f"Project {project.name} is added in project registry as it's a child project and not explicitly denied in `deny` list" + ) projects_to_ingest[project.id] = project # We rely on automatic browse paths (v2) when creating containers. That's why we need to sort the projects here. From 11c49ec6be3c2145151a91924799138d4b56bb3b Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 11 Dec 2024 17:31:06 -0500 Subject: [PATCH 04/16] fix(ui) Add backwards compatibility to the UI for old policy filters (#12017) --- .../permissions/policy/PolicyDetailsModal.tsx | 8 +- .../policy/PolicyPrivilegeForm.tsx | 29 +++- .../policy/_tests_/policyUtils.test.tsx | 160 +++++++++++++++++- .../src/app/permissions/policy/constants.ts | 4 + .../src/app/permissions/policy/policyUtils.ts | 24 ++- 5 files changed, 209 insertions(+), 16 deletions(-) create mode 100644 datahub-web-react/src/app/permissions/policy/constants.ts diff --git a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx index 37349585fa4c92..1988fea3496992 100644 --- a/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx +++ b/datahub-web-react/src/app/permissions/policy/PolicyDetailsModal.tsx @@ -12,6 +12,7 @@ import { mapResourceTypeToDisplayName, } from './policyUtils'; import AvatarsGroup from '../AvatarsGroup'; +import { RESOURCE_TYPE, RESOURCE_URN, TYPE, URN } from './constants'; type PrivilegeOptionType = { type?: string; @@ -72,10 +73,11 @@ export default function PolicyDetailsModal({ policy, open, onClose, privileges } const isMetadataPolicy = policy?.type === PolicyType.Metadata; const resources = convertLegacyResourceFilter(policy?.resources); - const resourceTypes = getFieldValues(resources?.filter, 'TYPE') || []; + const resourceTypes = getFieldValues(resources?.filter, TYPE, RESOURCE_TYPE) || []; const dataPlatformInstances = getFieldValues(resources?.filter, 'DATA_PLATFORM_INSTANCE') || []; - const resourceEntities = getFieldValues(resources?.filter, 'URN') || []; - const resourceFilterCondition = getFieldCondition(resources?.filter, 'URN') || PolicyMatchCondition.Equals; + const resourceEntities = getFieldValues(resources?.filter, URN, RESOURCE_URN) || []; + const resourceFilterCondition = + getFieldCondition(resources?.filter, URN, RESOURCE_URN) || PolicyMatchCondition.Equals; const domains = getFieldValues(resources?.filter, 'DOMAIN') || []; const { diff --git a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx index 7a0de67f414192..414346c2776db8 100644 --- a/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx +++ b/datahub-web-react/src/app/permissions/policy/PolicyPrivilegeForm.tsx @@ -28,6 +28,7 @@ import ClickOutside from '../../shared/ClickOutside'; import { TagTermLabel } from '../../shared/tags/TagTermLabel'; import { ENTER_KEY_CODE } from '../../shared/constants'; import { useGetRecommendations } from '../../shared/recommendation'; +import { RESOURCE_TYPE, RESOURCE_URN, TYPE, URN } from './constants'; type Props = { policyType: PolicyType; @@ -102,8 +103,9 @@ export default function PolicyPrivilegeForm({ } = useAppConfig(); const resources: ResourceFilter = convertLegacyResourceFilter(maybeResources) || EMPTY_POLICY.resources; - const resourceTypes = getFieldValues(resources.filter, 'TYPE') || []; - const resourceEntities = getFieldValues(resources.filter, 'URN') || []; + // RESOURCE_TYPE and RESOURCE_URN are deprecated, but need to get them for backwards compatibility + const resourceTypes = getFieldValues(resources.filter, TYPE, RESOURCE_TYPE) || []; + const resourceEntities = getFieldValues(resources.filter, URN, RESOURCE_URN) || []; const getDisplayName = (entity) => { if (!entity) { @@ -178,9 +180,14 @@ export default function PolicyPrivilegeForm({ const filter = resources.filter || { criteria: [], }; + // remove the deprecated RESOURCE_TYPE field and replace with TYPE field + const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_TYPE, []); setResources({ ...resources, - filter: setFieldValues(filter, 'TYPE', [...resourceTypes, createCriterionValue(selectedResourceType)]), + filter: setFieldValues(filterWithoutDeprecatedField, TYPE, [ + ...resourceTypes, + createCriterionValue(selectedResourceType), + ]), }); }; @@ -188,11 +195,13 @@ export default function PolicyPrivilegeForm({ const filter = resources.filter || { criteria: [], }; + // remove the deprecated RESOURCE_TYPE field and replace with TYPE field + const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_TYPE, []); setResources({ ...resources, filter: setFieldValues( - filter, - 'TYPE', + filterWithoutDeprecatedField, + TYPE, resourceTypes?.filter((criterionValue) => criterionValue.value !== deselectedResourceType), ), }); @@ -203,9 +212,11 @@ export default function PolicyPrivilegeForm({ const filter = resources.filter || { criteria: [], }; + // remove the deprecated RESOURCE_URN field and replace with URN field + const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_URN, []); setResources({ ...resources, - filter: setFieldValues(filter, 'URN', [ + filter: setFieldValues(filterWithoutDeprecatedField, URN, [ ...resourceEntities, createCriterionValueWithEntity( resource, @@ -220,11 +231,13 @@ export default function PolicyPrivilegeForm({ const filter = resources.filter || { criteria: [], }; + // remove the deprecated RESOURCE_URN field and replace with URN field + const filterWithoutDeprecatedField = setFieldValues(filter, RESOURCE_URN, []); setResources({ ...resources, filter: setFieldValues( - filter, - 'URN', + filterWithoutDeprecatedField, + URN, resourceEntities?.filter((criterionValue) => criterionValue.value !== resource), ), }); diff --git a/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx b/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx index 1c9884e5fcf09c..eae735b3477f0f 100644 --- a/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx +++ b/datahub-web-react/src/app/permissions/policy/_tests_/policyUtils.test.tsx @@ -1,4 +1,12 @@ -import { addOrUpdatePoliciesInList, updateListPoliciesCache, removeFromListPoliciesCache } from '../policyUtils'; +import { PolicyMatchCondition } from '../../../../types.generated'; +import { + addOrUpdatePoliciesInList, + updateListPoliciesCache, + removeFromListPoliciesCache, + getFieldValues, + getFieldCondition, + setFieldValues, +} from '../policyUtils'; // Mock the Apollo Client readQuery and writeQuery methods const mockReadQuery = vi.fn(); @@ -103,3 +111,153 @@ describe('removeFromListPoliciesCache', () => { }); }); }); + +describe('getFieldValues', () => { + it('should get field values for a given field', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataset' }, { value: 'dataJob' }], + }, + ], + }; + + expect(getFieldValues(filter, 'TYPE')).toMatchObject([{ value: 'dataset' }, { value: 'dataJob' }]); + }); + + it('should get field values for a alternate field (for deprecated fields)', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'RESOURCE_TYPE', + values: [{ value: 'dataset' }, { value: 'dataJob' }], + }, + ], + }; + + expect(getFieldValues(filter, 'TYPE', 'RESOURCE_TYPE')).toMatchObject([ + { value: 'dataset' }, + { value: 'dataJob' }, + ]); + }); + + it('should get field values for main field with alternative field given and has values', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'RESOURCE_TYPE', + values: [{ value: 'container' }, { value: 'dataFlow' }], + }, + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataset' }, { value: 'dataJob' }], + }, + ], + }; + + // should only return values from main field + expect(getFieldValues(filter, 'TYPE', 'RESOURCE_TYPE')).toMatchObject([ + { value: 'dataset' }, + { value: 'dataJob' }, + ]); + }); +}); + +describe('getFieldCondition', () => { + it('should get field values for a given field', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataset' }], + }, + ], + }; + + expect(getFieldCondition(filter, 'TYPE')).toBe(PolicyMatchCondition.Equals); + }); + + it('should get field values for a alternate field (for deprecated fields)', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'RESOURCE_TYPE', + values: [{ value: 'dataset' }], + }, + ], + }; + + expect(getFieldCondition(filter, 'TYPE', 'RESOURCE_TYPE')).toBe(PolicyMatchCondition.Equals); + }); + + it('should get field values for main field with alternative field given and has values', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.StartsWith, + field: 'RESOURCE_TYPE', + values: [{ value: 'container' }, { value: 'dataFlow' }], + }, + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataset' }], + }, + ], + }; + + // should only return values from main field + expect(getFieldCondition(filter, 'TYPE', 'RESOURCE_TYPE')).toBe(PolicyMatchCondition.Equals); + }); +}); +describe('setFieldValues', () => { + it('should remove a field if you pass in an empty array', () => { + const filter = { + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'RESOURCE_TYPE', + values: [{ value: 'dataset' }], + }, + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataJob' }], + }, + ], + }; + + expect(setFieldValues(filter, 'RESOURCE_TYPE', [])).toMatchObject({ + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataJob' }], + }, + ], + }); + }); + + it('should set values for a field properly', () => { + const filter = { + criteria: [], + }; + + expect(setFieldValues(filter, 'TYPE', [{ value: 'dataFlow' }])).toMatchObject({ + criteria: [ + { + condition: PolicyMatchCondition.Equals, + field: 'TYPE', + values: [{ value: 'dataFlow' }], + }, + ], + }); + }); +}); diff --git a/datahub-web-react/src/app/permissions/policy/constants.ts b/datahub-web-react/src/app/permissions/policy/constants.ts new file mode 100644 index 00000000000000..cdd20bf9b50d6a --- /dev/null +++ b/datahub-web-react/src/app/permissions/policy/constants.ts @@ -0,0 +1,4 @@ +export const TYPE = 'TYPE'; +export const RESOURCE_TYPE = 'RESOURCE_TYPE'; +export const URN = 'URN'; +export const RESOURCE_URN = 'RESOURCE_URN'; diff --git a/datahub-web-react/src/app/permissions/policy/policyUtils.ts b/datahub-web-react/src/app/permissions/policy/policyUtils.ts index b71a38f80fc256..d6221a0a9293af 100644 --- a/datahub-web-react/src/app/permissions/policy/policyUtils.ts +++ b/datahub-web-react/src/app/permissions/policy/policyUtils.ts @@ -114,12 +114,28 @@ export const convertLegacyResourceFilter = (resourceFilter: Maybe | undefined, resourceFieldType: string) => { - return filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.values || []; +export const getFieldValues = ( + filter: Maybe | undefined, + resourceFieldType: string, + alternateResourceFieldType?: string, +) => { + return ( + filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.values || + filter?.criteria?.find((criterion) => criterion.field === alternateResourceFieldType)?.values || + [] + ); }; -export const getFieldCondition = (filter: Maybe | undefined, resourceFieldType: string) => { - return filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.condition || null; +export const getFieldCondition = ( + filter: Maybe | undefined, + resourceFieldType: string, + alternateResourceFieldType?: string, +) => { + return ( + filter?.criteria?.find((criterion) => criterion.field === resourceFieldType)?.condition || + filter?.criteria?.find((criterion) => criterion.field === alternateResourceFieldType)?.condition || + null + ); }; export const getFieldValuesOfTags = (filter: Maybe | undefined, resourceFieldType: string) => { From 8b5fb711929e465c45c58e6db8c657c2623cb5e9 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Wed, 11 Dec 2024 18:45:46 -0500 Subject: [PATCH 05/16] feat(structuredProps) Add frontend for managing structured props and filtering by them (#12097) --- .../datahub/graphql/GmsGraphQLEngine.java | 14 + .../authorization/AuthorizationUtils.java | 5 + .../datahub/graphql/resolvers/MeResolver.java | 4 + .../resolvers/config/AppConfigResolver.java | 1 + .../AggregateAcrossEntitiesResolver.java | 8 +- .../search/SearchAcrossEntitiesResolver.java | 80 ++- .../graphql/resolvers/search/SearchUtils.java | 22 + .../graphql/types/mappers/MapperUtils.java | 3 + .../src/main/resources/app.graphql | 14 + .../src/main/resources/search.graphql | 10 + .../SearchAcrossEntitiesResolverTest.java | 6 +- datahub-web-react/src/Mocks.tsx | 66 ++- .../SearchBar/SearchBar.stories.tsx | 99 ++++ .../components/SearchBar/SearchBar.tsx | 30 ++ .../components/SearchBar/components.ts | 33 ++ .../components/SearchBar/index.ts | 1 + .../components/SearchBar/types.ts | 7 + .../src/alchemy-components/index.ts | 1 + datahub-web-react/src/app/SearchRoutes.tsx | 18 +- datahub-web-react/src/app/analytics/event.ts | 86 +++- .../src/app/buildEntityRegistry.ts | 2 + .../src/app/entity/Access/RoleEntity.tsx | 4 + datahub-web-react/src/app/entity/Entity.tsx | 5 + .../src/app/entity/EntityRegistry.tsx | 4 + .../BusinessAttributeEntity.tsx | 2 + .../src/app/entity/chart/ChartEntity.tsx | 2 + .../app/entity/container/ContainerEntity.tsx | 2 + .../app/entity/dashboard/DashboardEntity.tsx | 2 + .../app/entity/dataFlow/DataFlowEntity.tsx | 2 + .../src/app/entity/dataJob/DataJobEntity.tsx | 2 + .../dataPlatform/DataPlatformEntity.tsx | 4 + .../DataPlatformInstanceEntity.tsx | 4 + .../entity/dataProduct/DataProductEntity.tsx | 4 + .../src/app/entity/dataset/DatasetEntity.tsx | 2 + .../src/app/entity/domain/DomainEntity.tsx | 2 + .../ERModelRelationshipEntity.tsx | 2 + .../glossaryNode/GlossaryNodeEntity.tsx | 2 + .../glossaryTerm/GlossaryTermEntity.tsx | 2 + .../src/app/entity/group/Group.tsx | 2 + .../app/entity/mlFeature/MLFeatureEntity.tsx | 2 + .../mlFeatureTable/MLFeatureTableEntity.tsx | 2 + .../src/app/entity/mlModel/MLModelEntity.tsx | 2 + .../mlModelGroup/MLModelGroupEntity.tsx | 2 + .../mlPrimaryKey/MLPrimaryKeyEntity.tsx | 2 + .../SchemaFieldPropertiesEntity.tsx | 2 + .../Properties/useStructuredProperties.tsx | 19 + .../StructuredPropertyEntity.tsx | 86 ++++ datahub-web-react/src/app/entity/tag/Tag.tsx | 2 + .../src/app/entity/user/User.tsx | 2 + .../structuredProperties/AdvancedOptions.tsx | 64 +++ .../AllowedValuesDrawer.tsx | 142 ++++++ .../AllowedValuesField.tsx | 77 +++ .../DisplayPreferences.tsx | 165 ++++++ .../EmptyStructuredProperties.tsx | 29 ++ .../structuredProperties/RequiredAsterisk.tsx | 12 + .../StructuredProperties.tsx | 115 +++++ .../StructuredPropsDrawer.tsx | 396 +++++++++++++++ .../StructuredPropsForm.tsx | 139 +++++ .../StructuredPropsFormSection.tsx | 176 +++++++ .../StructuredPropsTable.tsx | 376 ++++++++++++++ .../ViewAdvancedOptions.tsx | 43 ++ .../ViewDisplayPreferences.tsx | 86 ++++ .../ViewStructuredPropsDrawer.tsx | 142 ++++++ .../govern/structuredProperties/cacheUtils.ts | 102 ++++ .../structuredProperties/styledComponents.ts | 473 ++++++++++++++++++ .../structuredProperties/useStructuredProp.ts | 135 +++++ .../app/govern/structuredProperties/utils.ts | 249 +++++++++ .../src/app/search/SearchPage.tsx | 2 +- .../src/app/search/filters/ActiveFilter.tsx | 11 +- .../filters/DateRangeMenu/DateRangeMenu.tsx | 88 ++++ .../DateRangeMenu/useDateRangeFilterValues.ts | 48 ++ .../EntityTypeFilter/entityTypeFilterUtils.ts | 2 +- .../src/app/search/filters/FilterOption.tsx | 9 +- .../app/search/filters/MoreFilterOption.tsx | 17 +- .../search/filters/OptionsDropdownMenu.tsx | 15 + .../src/app/search/filters/SearchFilter.tsx | 8 +- .../app/search/filters/SearchFilterView.tsx | 12 +- .../app/search/filters/mapFilterOption.tsx | 2 +- .../src/app/search/filters/types.ts | 1 + .../filters/useSearchFilterDropdown.tsx | 18 +- .../src/app/search/filters/utils.tsx | 62 ++- .../src/app/search/utils/constants.ts | 5 + .../src/app/shared/admin/HeaderLinks.tsx | 25 + datahub-web-react/src/app/shared/constants.ts | 3 + .../app/sharedV2/modals/ConfirmationModal.tsx | 76 +++ .../src/app/sharedV2/toastMessageUtils.ts | 18 + datahub-web-react/src/appConfigContext.tsx | 1 + datahub-web-react/src/conf/Global.ts | 1 + datahub-web-react/src/graphql/app.graphql | 1 + .../src/graphql/fragments.graphql | 55 ++ datahub-web-react/src/graphql/lineage.graphql | 35 ++ datahub-web-react/src/graphql/me.graphql | 2 + datahub-web-react/src/graphql/search.graphql | 90 ++++ .../src/graphql/structuredProperties.graphql | 137 +++++ datahub-web-react/src/images/empty-forms.svg | 48 ++ datahub-web-react/src/images/table-icon.svg | 3 + .../request/AggregationQueryBuilder.java | 54 +- .../fixtures/SampleDataFixtureTestBase.java | 17 +- .../request/AggregationQueryBuilderTest.java | 117 +++-- .../request/SearchRequestHandlerTest.java | 3 +- .../linkedin/metadata/query/SearchFlags.pdl | 6 + .../metadata/search/AggregationMetadata.pdl | 5 + .../graphql/featureflags/FeatureFlags.java | 1 + .../src/main/resources/application.yaml | 1 + ...linkedin.analytics.analytics.snapshot.json | 2 +- .../com.linkedin.entity.aspects.snapshot.json | 54 +- ...com.linkedin.entity.entities.snapshot.json | 75 ++- .../com.linkedin.entity.runs.snapshot.json | 52 +- ...nkedin.operations.operations.snapshot.json | 52 +- ...m.linkedin.platform.platform.snapshot.json | 60 ++- .../datahubusage/DataHubUsageEventType.java | 10 +- .../war/src/main/resources/boot/policies.json | 6 +- .../authorization/PoliciesConfig.java | 6 + 113 files changed, 4694 insertions(+), 223 deletions(-) create mode 100644 datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.stories.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.tsx create mode 100644 datahub-web-react/src/alchemy-components/components/SearchBar/components.ts create mode 100644 datahub-web-react/src/alchemy-components/components/SearchBar/index.ts create mode 100644 datahub-web-react/src/alchemy-components/components/SearchBar/types.ts create mode 100644 datahub-web-react/src/app/entity/structuredProperty/StructuredPropertyEntity.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/AdvancedOptions.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/AllowedValuesDrawer.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/AllowedValuesField.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/DisplayPreferences.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/EmptyStructuredProperties.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/RequiredAsterisk.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/StructuredProperties.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/StructuredPropsDrawer.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/StructuredPropsForm.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/StructuredPropsFormSection.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/StructuredPropsTable.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/ViewAdvancedOptions.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/ViewDisplayPreferences.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/ViewStructuredPropsDrawer.tsx create mode 100644 datahub-web-react/src/app/govern/structuredProperties/cacheUtils.ts create mode 100644 datahub-web-react/src/app/govern/structuredProperties/styledComponents.ts create mode 100644 datahub-web-react/src/app/govern/structuredProperties/useStructuredProp.ts create mode 100644 datahub-web-react/src/app/govern/structuredProperties/utils.ts create mode 100644 datahub-web-react/src/app/search/filters/DateRangeMenu/DateRangeMenu.tsx create mode 100644 datahub-web-react/src/app/search/filters/DateRangeMenu/useDateRangeFilterValues.ts create mode 100644 datahub-web-react/src/app/sharedV2/modals/ConfirmationModal.tsx create mode 100644 datahub-web-react/src/app/sharedV2/toastMessageUtils.ts create mode 100644 datahub-web-react/src/images/empty-forms.svg create mode 100644 datahub-web-react/src/images/table-icon.svg diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index d1da55268a50d5..079a20619d1eab 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -67,6 +67,7 @@ import com.linkedin.datahub.graphql.generated.EntityPath; import com.linkedin.datahub.graphql.generated.EntityRelationship; import com.linkedin.datahub.graphql.generated.EntityRelationshipLegacy; +import com.linkedin.datahub.graphql.generated.FacetMetadata; import com.linkedin.datahub.graphql.generated.ForeignKeyConstraint; import com.linkedin.datahub.graphql.generated.FormActorAssignment; import com.linkedin.datahub.graphql.generated.FreshnessContract; @@ -1474,6 +1475,19 @@ private void configureGenericEntityResolvers(final RuntimeWiring.Builder builder "entity", new EntityTypeResolver( entityTypes, (env) -> ((BrowsePathEntry) env.getSource()).getEntity()))) + .type( + "FacetMetadata", + typeWiring -> + typeWiring.dataFetcher( + "entity", + new EntityTypeResolver( + entityTypes, + (env) -> { + FacetMetadata facetMetadata = env.getSource(); + return facetMetadata.getEntity() != null + ? facetMetadata.getEntity() + : null; + }))) .type( "LineageRelationship", typeWiring -> diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java index ca60acaa805387..c25d6af75fe76d 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java @@ -339,6 +339,11 @@ public static boolean canManageStructuredProperties(@Nonnull QueryContext contex context.getOperationContext(), PoliciesConfig.MANAGE_STRUCTURED_PROPERTIES_PRIVILEGE); } + public static boolean canViewStructuredPropertiesPage(@Nonnull QueryContext context) { + return AuthUtil.isAuthorized( + context.getOperationContext(), PoliciesConfig.VIEW_STRUCTURED_PROPERTIES_PAGE_PRIVILEGE); + } + public static boolean canManageForms(@Nonnull QueryContext context) { return AuthUtil.isAuthorized( context.getOperationContext(), PoliciesConfig.MANAGE_DOCUMENTATION_FORMS_PRIVILEGE); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java index b1101ae3ee8657..8297392e642d51 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java @@ -93,6 +93,10 @@ public CompletableFuture get(DataFetchingEnvironment environm BusinessAttributeAuthorizationUtils.canCreateBusinessAttribute(context)); platformPrivileges.setManageBusinessAttributes( BusinessAttributeAuthorizationUtils.canManageBusinessAttribute(context)); + platformPrivileges.setManageStructuredProperties( + AuthorizationUtils.canManageStructuredProperties(context)); + platformPrivileges.setViewStructuredPropertiesPage( + AuthorizationUtils.canViewStructuredPropertiesPage(context)); // Construct and return authenticated user object. final AuthenticatedUser authUser = new AuthenticatedUser(); authUser.setCorpUser(corpUser); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index 259d05c631557d..3647eb55b2583a 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -188,6 +188,7 @@ public CompletableFuture get(final DataFetchingEnvironment environmen .setDataContractsEnabled(_featureFlags.isDataContractsEnabled()) .setEditableDatasetNameEnabled(_featureFlags.isEditableDatasetNameEnabled()) .setShowSeparateSiblings(_featureFlags.isShowSeparateSiblings()) + .setShowManageStructuredProperties(_featureFlags.isShowManageStructuredProperties()) .build(); appConfig.setFeatureFlags(featureFlagsConfig); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java index 29b71d95ad9749..31ed2de7a6d513 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/AggregateAcrossEntitiesResolver.java @@ -66,11 +66,17 @@ public CompletableFuture get(DataFetchingEnvironment environme final Filter inputFilter = ResolverUtils.buildFilter(null, input.getOrFilters()); - final SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags()); + final SearchFlags searchFlags = + input.getSearchFlags() != null + ? mapInputFlags(context, input.getSearchFlags()) + : new SearchFlags(); final List facets = input.getFacets() != null && input.getFacets().size() > 0 ? input.getFacets() : null; + // do not include default facets if we're requesting any facets specifically + searchFlags.setIncludeDefaultFacets(facets == null || facets.size() <= 0); + List finalEntities = maybeResolvedView != null ? SearchUtils.intersectEntityTypes( diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java index d103704146d399..29bc3a82a16498 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java @@ -2,19 +2,28 @@ import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.bindArgument; import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.*; +import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.getEntityNames; +import com.google.common.collect.ImmutableList; import com.linkedin.common.urn.UrnUtils; import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.concurrency.GraphQLConcurrencyUtils; +import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.SearchAcrossEntitiesInput; import com.linkedin.datahub.graphql.generated.SearchResults; import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.query.SearchFlags; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.CriterionArray; import com.linkedin.metadata.query.filter.Filter; import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.search.SearchResult; import com.linkedin.metadata.service.ViewService; +import com.linkedin.metadata.utils.CriterionUtils; import com.linkedin.view.DataHubViewInfo; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; @@ -64,24 +73,7 @@ public CompletableFuture get(DataFetchingEnvironment environment) ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); SearchFlags searchFlags = mapInputFlags(context, input.getSearchFlags()); - List sortCriteria; - if (input.getSortInput() != null) { - if (input.getSortInput().getSortCriteria() != null) { - sortCriteria = - input.getSortInput().getSortCriteria().stream() - .map(SearchUtils::mapSortCriterion) - .collect(Collectors.toList()); - } else { - sortCriteria = - input.getSortInput().getSortCriterion() != null - ? Collections.singletonList( - mapSortCriterion(input.getSortInput().getSortCriterion())) - : Collections.emptyList(); - } - - } else { - sortCriteria = Collections.emptyList(); - } + List sortCriteria = SearchUtils.getSortCriteria(input.getSortInput()); try { log.debug( @@ -101,6 +93,14 @@ public CompletableFuture get(DataFetchingEnvironment environment) return SearchUtils.createEmptySearchResults(start, count); } + boolean shouldIncludeStructuredPropertyFacets = + input.getSearchFlags() != null + && input.getSearchFlags().getIncludeStructuredPropertyFacets() != null + ? input.getSearchFlags().getIncludeStructuredPropertyFacets() + : false; + List structuredPropertyFacets = + shouldIncludeStructuredPropertyFacets ? getStructuredPropertyFacets(context) : null; + return UrnSearchResultsMapper.map( context, _entityClient.searchAcrossEntities( @@ -113,7 +113,8 @@ public CompletableFuture get(DataFetchingEnvironment environment) : baseFilter, start, count, - sortCriteria)); + sortCriteria, + structuredPropertyFacets)); } catch (Exception e) { log.error( "Failed to execute search for multiple entities: entity types {}, query {}, filters: {}, start: {}, count: {}", @@ -133,4 +134,45 @@ public CompletableFuture get(DataFetchingEnvironment environment) this.getClass().getSimpleName(), "get"); } + + private List getStructuredPropertyFacets(final QueryContext context) { + try { + SearchFlags searchFlags = new SearchFlags().setSkipCache(true); + SearchResult result = + _entityClient.searchAcrossEntities( + context.getOperationContext().withSearchFlags(flags -> searchFlags), + getEntityNames(ImmutableList.of(EntityType.STRUCTURED_PROPERTY)), + "*", + createStructuredPropertyFilter(), + 0, + 100, + Collections.emptyList(), + null); + return result.getEntities().stream() + .map(entity -> String.format("structuredProperties.%s", entity.getEntity().getId())) + .collect(Collectors.toList()); + } catch (Exception e) { + log.error("Failed to get structured property facets to filter on", e); + return Collections.emptyList(); + } + } + + private Filter createStructuredPropertyFilter() { + return new Filter() + .setOr( + new ConjunctiveCriterionArray( + ImmutableList.of( + new ConjunctiveCriterion() + .setAnd( + new CriterionArray( + ImmutableList.of( + CriterionUtils.buildCriterion( + "filterStatus", Condition.EQUAL, "ENABLED")))), + new ConjunctiveCriterion() + .setAnd( + new CriterionArray( + ImmutableList.of( + CriterionUtils.buildCriterion( + "showInSearchFilters", Condition.EQUAL, "true"))))))); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java index 04777c3fcdb4e2..a01b3aaec9c982 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java @@ -22,6 +22,7 @@ import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.FacetFilterInput; import com.linkedin.datahub.graphql.generated.SearchResults; +import com.linkedin.datahub.graphql.generated.SearchSortInput; import com.linkedin.datahub.graphql.types.common.mappers.SearchFlagsInputMapper; import com.linkedin.datahub.graphql.types.entitytype.EntityTypeMapper; import com.linkedin.metadata.query.SearchFlags; @@ -326,4 +327,25 @@ public static SearchResults createEmptySearchResults(final int start, final int result.setFacets(new ArrayList<>()); return result; } + + public static List getSortCriteria(@Nullable final SearchSortInput sortInput) { + List sortCriteria; + if (sortInput != null) { + if (sortInput.getSortCriteria() != null) { + sortCriteria = + sortInput.getSortCriteria().stream() + .map(SearchUtils::mapSortCriterion) + .collect(Collectors.toList()); + } else { + sortCriteria = + sortInput.getSortCriterion() != null + ? Collections.singletonList(mapSortCriterion(sortInput.getSortCriterion())) + : new ArrayList<>(); + } + } else { + sortCriteria = new ArrayList<>(); + } + + return sortCriteria; + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java index 0d69e62c621a60..8fe58df2d2edec 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/mappers/MapperUtils.java @@ -70,6 +70,9 @@ public static FacetMetadata mapFacet( aggregationFacets.stream() .map(facet -> facet.equals("entity") || facet.contains("_entityType")) .collect(Collectors.toList()); + if (aggregationMetadata.getEntity() != null) { + facetMetadata.setEntity(UrnToEntityMapper.map(context, aggregationMetadata.getEntity())); + } facetMetadata.setField(aggregationMetadata.getName()); facetMetadata.setDisplayName( Optional.ofNullable(aggregationMetadata.getDisplayName()) diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index 262d2384d84ada..28688903687235 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -156,6 +156,15 @@ type PlatformPrivileges { """ manageBusinessAttributes: Boolean! + """ + Whether the user can create, edit, and delete structured properties. + """ + manageStructuredProperties: Boolean! + + """ + Whether the user can view the manage structured properties page. + """ + viewStructuredPropertiesPage: Boolean! } """ @@ -517,6 +526,11 @@ type FeatureFlagsConfig { If turned on, all siblings will be separated with no way to get to a "combined" sibling view """ showSeparateSiblings: Boolean! + + """ + If turned on, show the manage structured properties tab in the govern dropdown + """ + showManageStructuredProperties: Boolean! } """ diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql index d0f669f05f9598..82bfb9ee26fc42 100644 --- a/datahub-graphql-core/src/main/resources/search.graphql +++ b/datahub-graphql-core/src/main/resources/search.graphql @@ -167,6 +167,11 @@ input SearchFlags { fields to include for custom Highlighting """ customHighlightingFields: [String!] + + """ + Whether or not to fetch and request for structured property facets when doing a search + """ + includeStructuredPropertyFacets: Boolean } """ @@ -872,6 +877,11 @@ type FacetMetadata { """ displayName: String + """ + Entity corresponding to the facet + """ + entity: Entity + """ Aggregated search result counts by value of the field """ diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java index 42768b8a2de21b..89d218683e33ec 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java @@ -471,7 +471,8 @@ private static EntityClient initMockEntityClient( Mockito.eq(filter), Mockito.eq(start), Mockito.eq(limit), - Mockito.eq(Collections.emptyList()))) + Mockito.eq(Collections.emptyList()), + Mockito.eq(null))) .thenReturn(result); return client; } @@ -496,7 +497,8 @@ private static void verifyMockEntityClient( Mockito.eq(filter), Mockito.eq(start), Mockito.eq(limit), - Mockito.eq(Collections.emptyList())); + Mockito.eq(Collections.emptyList()), + Mockito.eq(null)); } private static void verifyMockViewService(ViewService mockService, Urn viewUrn) { diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index 329d6250e576ab..73a789030ce6fb 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -2204,7 +2204,7 @@ export const mocks = [ count: 10, filters: [], orFilters: [], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -2244,6 +2244,7 @@ export const mocks = [ field: 'origin', displayName: 'origin', aggregations: [{ value: 'PROD', count: 3, entity: null }], + entity: null, }, { field: '_entityType', @@ -2252,6 +2253,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -2261,6 +2263,7 @@ export const mocks = [ { value: 'MySQL', count: 1, entity: null }, { value: 'Kafka', count: 1, entity: null }, ], + entity: null, }, ], suggestions: [], @@ -2290,7 +2293,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -2325,6 +2328,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -2333,6 +2337,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -2343,6 +2348,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], suggestions: [], @@ -2393,6 +2399,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -2401,6 +2408,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -2410,6 +2418,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -2464,7 +2473,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -2501,6 +2510,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -2510,6 +2520,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -2520,6 +2531,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null, __typename: 'AggregationMetadata' }, { value: 'kafka', count: 1, entity: null, __typename: 'AggregationMetadata' }, ], + entity: null, }, ], }, @@ -2669,6 +2681,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -2677,6 +2690,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -2686,6 +2700,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -2743,6 +2758,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -2751,6 +2767,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -2760,6 +2777,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -2809,6 +2827,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: 'platform', @@ -2822,6 +2841,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -2953,6 +2973,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: 'platform', @@ -2966,6 +2987,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -3013,7 +3035,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3050,6 +3072,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, // { // displayName: 'Domain', @@ -3071,6 +3094,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -3096,6 +3120,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, ], }, @@ -3181,7 +3206,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3215,6 +3240,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: 'platform', @@ -3228,6 +3254,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -3256,7 +3283,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3290,6 +3317,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -3298,6 +3326,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -3307,6 +3336,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -3335,7 +3365,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3377,6 +3407,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -3385,6 +3416,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -3394,6 +3426,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -3428,7 +3461,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3465,6 +3498,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -3474,6 +3508,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -3484,6 +3519,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null, __typename: 'AggregationMetadata' }, { value: 'kafka', count: 1, entity: null, __typename: 'AggregationMetadata' }, ], + entity: null, }, ], }, @@ -3518,7 +3554,7 @@ export const mocks = [ ], }, ], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3555,6 +3591,7 @@ export const mocks = [ __typename: 'AggregationMetadata', }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -3564,6 +3601,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { __typename: 'FacetMetadata', @@ -3574,6 +3612,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null, __typename: 'AggregationMetadata' }, { value: 'kafka', count: 1, entity: null, __typename: 'AggregationMetadata' }, ], + entity: null, }, ], }, @@ -3635,6 +3674,8 @@ export const mocks = [ manageGlobalAnnouncements: true, createBusinessAttributes: true, manageBusinessAttributes: true, + manageStructuredProperties: true, + viewStructuredPropertiesPage: true, }, }, }, @@ -3722,7 +3763,7 @@ export const mocks = [ count: 10, filters: [], orFilters: [], - searchFlags: { getSuggestions: true }, + searchFlags: { getSuggestions: true, includeStructuredPropertyFacets: true }, }, }, }, @@ -3821,6 +3862,7 @@ export const mocks = [ entity: null, }, ], + entity: null, }, { field: '_entityType', @@ -3829,6 +3871,7 @@ export const mocks = [ { count: 37, entity: null, value: 'DATASET', __typename: 'AggregationMetadata' }, { count: 7, entity: null, value: 'CHART', __typename: 'AggregationMetadata' }, ], + entity: null, }, { field: 'platform', @@ -3838,6 +3881,7 @@ export const mocks = [ { value: 'mysql', count: 1, entity: null }, { value: 'kafka', count: 1, entity: null }, ], + entity: null, }, ], }, @@ -3912,4 +3956,6 @@ export const platformPrivileges: PlatformPrivileges = { manageGlobalAnnouncements: true, createBusinessAttributes: true, manageBusinessAttributes: true, + manageStructuredProperties: true, + viewStructuredPropertiesPage: true, }; diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.stories.tsx b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.stories.tsx new file mode 100644 index 00000000000000..9ae34356a71d6a --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.stories.tsx @@ -0,0 +1,99 @@ +import React, { useState } from 'react'; +import { BADGE } from '@geometricpanda/storybook-addon-badges'; +import type { Meta, StoryObj } from '@storybook/react'; +import { GridList } from '@components/.docs/mdx-components'; +import { SearchBar, searchBarDefaults } from './SearchBar'; +import { SearchBarProps } from './types'; + +const meta = { + title: 'Components / Search Bar', + component: SearchBar, + + // Display Properties + parameters: { + layout: 'centered', + badges: [BADGE.STABLE, 'readyForDesignReview'], + docs: { + subtitle: 'A component that is used to get search bar', + }, + }, + + // Component-level argTypes + argTypes: { + placeholder: { + description: 'Placeholder of search bar.', + table: { + defaultValue: { summary: searchBarDefaults.placeholder }, + }, + control: { + type: 'text', + }, + }, + value: { + description: 'Value of the search bar.', + table: { + defaultValue: { summary: searchBarDefaults.value }, + }, + control: false, + }, + width: { + description: 'Width of the search bar.', + table: { + defaultValue: { summary: searchBarDefaults.width }, + }, + control: { + type: 'text', + }, + }, + allowClear: { + description: 'Whether clear button should be present.', + table: { + defaultValue: { summary: searchBarDefaults.allowClear?.toString() }, + }, + control: { + type: 'boolean', + }, + }, + onChange: { + description: 'On change function for the search bar.', + }, + }, + + // Define defaults + args: { + placeholder: searchBarDefaults.placeholder, + value: searchBarDefaults.value, + allowClear: searchBarDefaults.allowClear, + width: searchBarDefaults.width, + }, +} satisfies Meta; + +export default meta; + +type Story = StoryObj; + +const SandboxWrapper = (props: SearchBarProps) => { + const [value, setValue] = useState(''); + + const handleChange = (newValue: string) => { + setValue(newValue); + }; + + return ; +}; + +export const sandbox: Story = { + tags: ['dev'], + render: (props) => { + return ; + }, +}; + +export const customWidths = () => ( + + + + + + +); diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.tsx b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.tsx new file mode 100644 index 00000000000000..f39f761058d8c7 --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/SearchBar/SearchBar.tsx @@ -0,0 +1,30 @@ +import { SearchOutlined } from '@ant-design/icons'; +import React from 'react'; +import { StyledSearchBar } from './components'; +import { SearchBarProps } from './types'; + +export const searchBarDefaults: SearchBarProps = { + placeholder: 'Search..', + value: '', + width: '272px', + allowClear: true, +}; + +export const SearchBar = ({ + placeholder = searchBarDefaults.placeholder, + value = searchBarDefaults.value, + width = searchBarDefaults.width, + allowClear = searchBarDefaults.allowClear, + onChange, +}: SearchBarProps) => { + return ( + onChange?.(e.target.value)} + value={value} + prefix={} + allowClear={allowClear} + $width={width} + /> + ); +}; diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/components.ts b/datahub-web-react/src/alchemy-components/components/SearchBar/components.ts new file mode 100644 index 00000000000000..7045801ddf092b --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/SearchBar/components.ts @@ -0,0 +1,33 @@ +import { colors, typography } from '@src/alchemy-components/theme'; +import { Input } from 'antd'; +import styled from 'styled-components'; + +export const StyledSearchBar = styled(Input)<{ $width?: string }>` + height: 40px; + width: ${(props) => props.$width}; + display: flex; + align-items: center; + border-radius: 8px; + + input { + color: ${colors.gray[600]}; + font-size: ${typography.fontSizes.md} !important; + } + + .ant-input-prefix { + width: 20px; + color: ${colors.gray[1800]}; + + svg { + height: 16px; + width: 16px; + } + } + + &:hover, + &:focus, + &:focus-within { + border-color: ${colors.violet[300]} !important; + box-shadow: none !important; + } +`; diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/index.ts b/datahub-web-react/src/alchemy-components/components/SearchBar/index.ts new file mode 100644 index 00000000000000..8c1933163b29fd --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/SearchBar/index.ts @@ -0,0 +1 @@ +export { SearchBar } from './SearchBar'; diff --git a/datahub-web-react/src/alchemy-components/components/SearchBar/types.ts b/datahub-web-react/src/alchemy-components/components/SearchBar/types.ts new file mode 100644 index 00000000000000..04ac218dc2cc6c --- /dev/null +++ b/datahub-web-react/src/alchemy-components/components/SearchBar/types.ts @@ -0,0 +1,7 @@ +export interface SearchBarProps { + placeholder?: string; + value?: string; + width?: string; + onChange?: (value: string) => void; + allowClear?: boolean; +} diff --git a/datahub-web-react/src/alchemy-components/index.ts b/datahub-web-react/src/alchemy-components/index.ts index 8ef4f73f4408ff..7e40d343e884dc 100644 --- a/datahub-web-react/src/alchemy-components/index.ts +++ b/datahub-web-react/src/alchemy-components/index.ts @@ -15,6 +15,7 @@ export * from './components/LineChart'; export * from './components/PageTitle'; export * from './components/Pills'; export * from './components/Popover'; +export * from './components/SearchBar'; export * from './components/Select'; export * from './components/Switch'; export * from './components/Table'; diff --git a/datahub-web-react/src/app/SearchRoutes.tsx b/datahub-web-react/src/app/SearchRoutes.tsx index 3343260c72bcf6..024b6e0add15a3 100644 --- a/datahub-web-react/src/app/SearchRoutes.tsx +++ b/datahub-web-react/src/app/SearchRoutes.tsx @@ -11,23 +11,36 @@ import { AnalyticsPage } from './analyticsDashboard/components/AnalyticsPage'; import { ManageIngestionPage } from './ingest/ManageIngestionPage'; import GlossaryRoutes from './glossary/GlossaryRoutes'; import { SettingsPage } from './settings/SettingsPage'; +import { useUserContext } from './context/useUserContext'; import DomainRoutes from './domain/DomainRoutes'; -import { useBusinessAttributesFlag, useIsAppConfigContextLoaded, useIsNestedDomainsEnabled } from './useAppConfig'; +import { + useAppConfig, + useBusinessAttributesFlag, + useIsAppConfigContextLoaded, + useIsNestedDomainsEnabled, +} from './useAppConfig'; import { ManageDomainsPage } from './domain/ManageDomainsPage'; import { BusinessAttributes } from './businessAttribute/BusinessAttributes'; +import StructuredProperties from './govern/structuredProperties/StructuredProperties'; /** * Container for all searchable page routes */ export const SearchRoutes = (): JSX.Element => { const entityRegistry = useEntityRegistry(); + const me = useUserContext(); const isNestedDomainsEnabled = useIsNestedDomainsEnabled(); const entities = isNestedDomainsEnabled ? entityRegistry.getEntitiesForSearchRoutes() : entityRegistry.getNonGlossaryEntities(); + const { config } = useAppConfig(); const businessAttributesFlag = useBusinessAttributesFlag(); const appConfigContextLoaded = useIsAppConfigContextLoaded(); + const showStructuredProperties = + config?.featureFlags?.showManageStructuredProperties && + (me.platformPrivileges?.manageStructuredProperties || me.platformPrivileges?.viewStructuredPropertiesPage); + return ( @@ -53,6 +66,9 @@ export const SearchRoutes = (): JSX.Element => { } /> } /> } /> + {showStructuredProperties && ( + } /> + )} { diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts index c3a57830b8c504..9152f2fb8eedb8 100644 --- a/datahub-web-react/src/app/analytics/event.ts +++ b/datahub-web-react/src/app/analytics/event.ts @@ -1,4 +1,12 @@ -import { DataHubViewType, EntityType, RecommendationRenderType, ScenarioType } from '../../types.generated'; +import { + AllowedValue, + DataHubViewType, + EntityType, + PropertyCardinality, + PropertyValueInput, + RecommendationRenderType, + ScenarioType, +} from '../../types.generated'; import { EmbedLookupNotFoundReason } from '../embed/lookup/constants'; import { Direction } from '../lineage/types'; import { FilterMode } from '../search/utils/constants'; @@ -82,6 +90,14 @@ export enum EventType { EmbedProfileViewInDataHubEvent, EmbedLookupNotFoundEvent, CreateBusinessAttributeEvent, + CreateStructuredPropertyClickEvent, + CreateStructuredPropertyEvent, + EditStructuredPropertyEvent, + DeleteStructuredPropertyEvent, + ViewStructuredPropertyEvent, + ApplyStructuredPropertyEvent, + UpdateStructuredPropertyOnAssetEvent, + RemoveStructuredPropertyEvent, } /** @@ -640,6 +656,64 @@ export interface CreateBusinessAttributeEvent extends BaseEvent { name: string; } +export interface CreateStructuredPropertyClickEvent extends BaseEvent { + type: EventType.CreateStructuredPropertyClickEvent; +} + +interface StructuredPropertyEvent extends BaseEvent { + propertyType: string; + appliesTo: string[]; + qualifiedName?: string; + allowedAssetTypes?: string[]; + allowedValues?: AllowedValue[]; + cardinality?: PropertyCardinality; + showInFilters?: boolean; + isHidden: boolean; + showInSearchFilters: boolean; + showAsAssetBadge: boolean; + showInAssetSummary: boolean; + showInColumnsTable: boolean; +} + +export interface CreateStructuredPropertyEvent extends StructuredPropertyEvent { + type: EventType.CreateStructuredPropertyEvent; +} + +export interface EditStructuredPropertyEvent extends StructuredPropertyEvent { + type: EventType.EditStructuredPropertyEvent; + propertyUrn: string; +} + +export interface DeleteStructuredPropertyEvent extends StructuredPropertyEvent { + type: EventType.DeleteStructuredPropertyEvent; + propertyUrn: string; +} + +export interface ViewStructuredPropertyEvent extends BaseEvent { + type: EventType.ViewStructuredPropertyEvent; + propertyUrn: string; +} + +interface StructuredPropertyOnAssetEvent extends BaseEvent { + propertyUrn: string; + propertyType: string; + assetUrn: string; + assetType: EntityType; +} +export interface ApplyStructuredPropertyEvent extends StructuredPropertyOnAssetEvent { + type: EventType.ApplyStructuredPropertyEvent; + values: PropertyValueInput[]; +} + +export interface UpdateStructuredPropertyOnAssetEvent extends StructuredPropertyOnAssetEvent { + type: EventType.UpdateStructuredPropertyOnAssetEvent; + values: PropertyValueInput[]; +} + +export interface RemoveStructuredPropertyEvent extends StructuredPropertyOnAssetEvent { + type: EventType.RemoveStructuredPropertyEvent; +} + /** * Event consisting of a union of specific event types. */ @@ -718,4 +792,12 @@ export type Event = | EmbedProfileViewEvent | EmbedProfileViewInDataHubEvent | EmbedLookupNotFoundEvent - | CreateBusinessAttributeEvent; + | CreateBusinessAttributeEvent + | CreateStructuredPropertyClickEvent + | CreateStructuredPropertyEvent + | EditStructuredPropertyEvent + | DeleteStructuredPropertyEvent + | ViewStructuredPropertyEvent + | ApplyStructuredPropertyEvent + | UpdateStructuredPropertyOnAssetEvent + | RemoveStructuredPropertyEvent; diff --git a/datahub-web-react/src/app/buildEntityRegistry.ts b/datahub-web-react/src/app/buildEntityRegistry.ts index 0b70986672be51..181ec7d328a587 100644 --- a/datahub-web-react/src/app/buildEntityRegistry.ts +++ b/datahub-web-react/src/app/buildEntityRegistry.ts @@ -24,6 +24,7 @@ import { RoleEntity } from './entity/Access/RoleEntity'; import { RestrictedEntity } from './entity/restricted/RestrictedEntity'; import { BusinessAttributeEntity } from './entity/businessAttribute/BusinessAttributeEntity'; import { SchemaFieldPropertiesEntity } from './entity/schemaField/SchemaFieldPropertiesEntity'; +import { StructuredPropertyEntity } from './entity/structuredProperty/StructuredPropertyEntity'; export default function buildEntityRegistry() { const registry = new EntityRegistry(); @@ -52,5 +53,6 @@ export default function buildEntityRegistry() { registry.register(new RestrictedEntity()); registry.register(new BusinessAttributeEntity()); registry.register(new SchemaFieldPropertiesEntity()); + registry.register(new StructuredPropertyEntity()); return registry; } diff --git a/datahub-web-react/src/app/entity/Access/RoleEntity.tsx b/datahub-web-react/src/app/entity/Access/RoleEntity.tsx index ab609b04f104ac..58a1ba8dd793b8 100644 --- a/datahub-web-react/src/app/entity/Access/RoleEntity.tsx +++ b/datahub-web-react/src/app/entity/Access/RoleEntity.tsx @@ -88,4 +88,8 @@ export class RoleEntity implements Entity { supportedCapabilities = () => { return new Set([EntityCapabilityType.OWNERS]); }; + + getGraphName = () => { + return 'roleEntity'; + }; } diff --git a/datahub-web-react/src/app/entity/Entity.tsx b/datahub-web-react/src/app/entity/Entity.tsx index 490f23330c5945..c56c97454a1d57 100644 --- a/datahub-web-react/src/app/entity/Entity.tsx +++ b/datahub-web-react/src/app/entity/Entity.tsx @@ -172,6 +172,11 @@ export interface Entity { */ getGenericEntityProperties: (data: T) => GenericEntityProperties | null; + /** + * Returns the graph name of the entity, as it appears in the GMS entity registry + */ + getGraphName: () => string; + /** * Returns the supported features for the entity */ diff --git a/datahub-web-react/src/app/entity/EntityRegistry.tsx b/datahub-web-react/src/app/entity/EntityRegistry.tsx index 00e7385ff5784b..0f65390f959df2 100644 --- a/datahub-web-react/src/app/entity/EntityRegistry.tsx +++ b/datahub-web-react/src/app/entity/EntityRegistry.tsx @@ -241,4 +241,8 @@ export default class EntityRegistry { const entity = validatedGet(type, this.entityTypeToEntity); return entity.getCustomCardUrlPath?.(); } + + getGraphNameFromType(type: EntityType): string { + return validatedGet(type, this.entityTypeToEntity).getGraphName(); + } } diff --git a/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx b/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx index b827a3c37d6a5c..442aaf735575ab 100644 --- a/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx +++ b/datahub-web-react/src/app/entity/businessAttribute/BusinessAttributeEntity.tsx @@ -59,6 +59,8 @@ export class BusinessAttributeEntity implements Entity { getCollectionName = () => 'Business Attributes'; + getGraphName = () => 'businessAttribute'; + getCustomCardUrlPath = () => PageRoutes.BUSINESS_ATTRIBUTE; isBrowseEnabled = () => false; diff --git a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx index 913d502972fe14..8a62a9018661e1 100644 --- a/datahub-web-react/src/app/entity/chart/ChartEntity.tsx +++ b/datahub-web-react/src/app/entity/chart/ChartEntity.tsx @@ -69,6 +69,8 @@ export class ChartEntity implements Entity { getAutoCompleteFieldName = () => 'title'; + getGraphName = () => 'chart'; + getPathName = () => 'chart'; getEntityName = () => 'Chart'; diff --git a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx index 89f9122c6287f8..941e7fc3f552da 100644 --- a/datahub-web-react/src/app/entity/container/ContainerEntity.tsx +++ b/datahub-web-react/src/app/entity/container/ContainerEntity.tsx @@ -59,6 +59,8 @@ export class ContainerEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'container'; + getPathName = () => 'container'; getEntityName = () => 'Container'; diff --git a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx index 9564cbc18198e4..95d4431d591790 100644 --- a/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx +++ b/datahub-web-react/src/app/entity/dashboard/DashboardEntity.tsx @@ -291,6 +291,8 @@ export class DashboardEntity implements Entity { ]); }; + getGraphName = () => this.getPathName(); + renderEmbeddedProfile = (urn: string) => ( { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'dataFlow'; + getPathName = () => 'pipelines'; getEntityName = () => 'Pipeline'; diff --git a/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx b/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx index fe1a906371e9d0..6bf95482269190 100644 --- a/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx +++ b/datahub-web-react/src/app/entity/dataJob/DataJobEntity.tsx @@ -64,6 +64,8 @@ export class DataJobEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'dataJob'; + getPathName = () => 'tasks'; getEntityName = () => 'Task'; diff --git a/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx b/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx index 6687ec9f914c1b..89cbaf3cbeaa12 100644 --- a/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx +++ b/datahub-web-react/src/app/entity/dataPlatform/DataPlatformEntity.tsx @@ -71,4 +71,8 @@ export class DataPlatformEntity implements Entity { supportedCapabilities = () => { return new Set([]); }; + + getGraphName = () => { + return 'dataPlatform'; + }; } diff --git a/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx b/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx index a542e1b52f510d..d0db687ffed929 100644 --- a/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx +++ b/datahub-web-react/src/app/entity/dataPlatformInstance/DataPlatformInstanceEntity.tsx @@ -58,4 +58,8 @@ export class DataPlatformInstanceEntity implements Entity supportedCapabilities = () => { return new Set([]); }; + + getGraphName = () => { + return 'dataPlatformInstance'; + }; } diff --git a/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx b/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx index 6b31de84f85bb1..90c1127d9a5fcd 100644 --- a/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx +++ b/datahub-web-react/src/app/entity/dataProduct/DataProductEntity.tsx @@ -191,4 +191,8 @@ export class DataProductEntity implements Entity { EntityCapabilityType.DOMAINS, ]); }; + + getGraphName = () => { + return 'dataProduct'; + }; } diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx index 21ae085832cb3f..07ab27a38f8893 100644 --- a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx +++ b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx @@ -85,6 +85,8 @@ export class DatasetEntity implements Entity { getPathName = () => 'dataset'; + getGraphName = () => 'dataset'; + getEntityName = () => 'Dataset'; getCollectionName = () => 'Datasets'; diff --git a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx index 2b67c88a6ff235..81d245c230843f 100644 --- a/datahub-web-react/src/app/entity/domain/DomainEntity.tsx +++ b/datahub-web-react/src/app/entity/domain/DomainEntity.tsx @@ -60,6 +60,8 @@ export class DomainEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'domain'; + getPathName = () => 'domain'; getEntityName = () => 'Domain'; diff --git a/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx b/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx index aece3db1312afb..3eb950cb0e7ac5 100644 --- a/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx +++ b/datahub-web-react/src/app/entity/ermodelrelationships/ERModelRelationshipEntity.tsx @@ -58,6 +58,8 @@ export class ERModelRelationshipEntity implements Entity { getEntityName = () => 'ER-Model-Relationship'; + getGraphName = () => 'erModelRelationship'; + renderProfile = (urn: string) => ( { EntityCapabilityType.SOFT_DELETE, ]); }; + + getGraphName = () => this.getPathName(); } export default GlossaryNodeEntity; diff --git a/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx b/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx index 8bbc0a693b2231..73c5a8e12122d3 100644 --- a/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx +++ b/datahub-web-react/src/app/entity/glossaryTerm/GlossaryTermEntity.tsx @@ -179,4 +179,6 @@ export class GlossaryTermEntity implements Entity { EntityCapabilityType.SOFT_DELETE, ]); }; + + getGraphName = () => this.getPathName(); } diff --git a/datahub-web-react/src/app/entity/group/Group.tsx b/datahub-web-react/src/app/entity/group/Group.tsx index cd9cf1ca6eec44..763db856f33ac6 100644 --- a/datahub-web-react/src/app/entity/group/Group.tsx +++ b/datahub-web-react/src/app/entity/group/Group.tsx @@ -40,6 +40,8 @@ export class GroupEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName: () => string = () => 'corpGroup'; + getPathName: () => string = () => 'group'; getEntityName = () => 'Group'; diff --git a/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx b/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx index 2f2786b1c0d960..eecffdb2f38430 100644 --- a/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx +++ b/datahub-web-react/src/app/entity/mlFeature/MLFeatureEntity.tsx @@ -52,6 +52,8 @@ export class MLFeatureEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'mlFeature'; + getPathName = () => 'features'; getEntityName = () => 'Feature'; diff --git a/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx b/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx index 595c73fbc3cb67..8aa0c056b716fc 100644 --- a/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx +++ b/datahub-web-react/src/app/entity/mlFeatureTable/MLFeatureTableEntity.tsx @@ -53,6 +53,8 @@ export class MLFeatureTableEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'mlFeatureTable'; + getPathName = () => 'featureTables'; getEntityName = () => 'Feature Table'; diff --git a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx index d4d0b37da9ec96..92f03aaef7a175 100644 --- a/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx +++ b/datahub-web-react/src/app/entity/mlModel/MLModelEntity.tsx @@ -52,6 +52,8 @@ export class MLModelEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'mlModel'; + getPathName = () => 'mlModels'; getEntityName = () => 'ML Model'; diff --git a/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx b/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx index 5896c1864cc435..b5d32275f97bfd 100644 --- a/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx +++ b/datahub-web-react/src/app/entity/mlModelGroup/MLModelGroupEntity.tsx @@ -50,6 +50,8 @@ export class MLModelGroupEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'mlModelGroup'; + getPathName = () => 'mlModelGroup'; getEntityName = () => 'ML Group'; diff --git a/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx b/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx index 60c7531a4f57cc..119a566b04f135 100644 --- a/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx +++ b/datahub-web-react/src/app/entity/mlPrimaryKey/MLPrimaryKeyEntity.tsx @@ -51,6 +51,8 @@ export class MLPrimaryKeyEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'mlPrimaryKey'; + getPathName = () => 'mlPrimaryKeys'; getEntityName = () => 'ML Primary Key'; diff --git a/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx b/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx index 88743012ddbc8a..2c59c476195d0b 100644 --- a/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx +++ b/datahub-web-react/src/app/entity/schemaField/SchemaFieldPropertiesEntity.tsx @@ -44,6 +44,8 @@ export class SchemaFieldPropertiesEntity implements Entity { // Currently unused. renderProfile = (_: string) => <>; + getGraphName = () => 'schemaField'; + renderPreview = (previewType: PreviewType, data: SchemaFieldEntity) => { const parent = data.parent as Dataset; return ( diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx index 86365b8232905c..4635486c24d1d6 100644 --- a/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx +++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/useStructuredProperties.tsx @@ -23,6 +23,25 @@ export function mapStructuredPropertyValues(structuredPropertiesEntry: Structure })); } +export function mapStructuredPropertyToPropertyRow(structuredPropertiesEntry: StructuredPropertiesEntry) { + const { displayName, qualifiedName } = structuredPropertiesEntry.structuredProperty.definition; + return { + displayName: displayName || qualifiedName, + qualifiedName, + values: mapStructuredPropertyValues(structuredPropertiesEntry), + dataType: structuredPropertiesEntry.structuredProperty.definition.valueType, + structuredProperty: structuredPropertiesEntry.structuredProperty, + type: + structuredPropertiesEntry.values[0] && structuredPropertiesEntry.values[0].__typename + ? { + type: typeNameToType[structuredPropertiesEntry.values[0].__typename].type, + nativeDataType: typeNameToType[structuredPropertiesEntry.values[0].__typename].nativeDataType, + } + : undefined, + associatedUrn: structuredPropertiesEntry.associatedUrn, + }; +} + // map the properties map into a list of PropertyRow objects to render in a table function getStructuredPropertyRows(entityData?: GenericEntityProperties | null) { const structuredPropertyRows: PropertyRow[] = []; diff --git a/datahub-web-react/src/app/entity/structuredProperty/StructuredPropertyEntity.tsx b/datahub-web-react/src/app/entity/structuredProperty/StructuredPropertyEntity.tsx new file mode 100644 index 00000000000000..2bdfd550652fa5 --- /dev/null +++ b/datahub-web-react/src/app/entity/structuredProperty/StructuredPropertyEntity.tsx @@ -0,0 +1,86 @@ +import * as React from 'react'; +import styled from 'styled-components'; +import TableIcon from '@src/images/table-icon.svg?react'; +import { TYPE_ICON_CLASS_NAME } from '@src/app/shared/constants'; +import DefaultPreviewCard from '@src/app/preview/DefaultPreviewCard'; +import { EntityType, SearchResult, StructuredPropertyEntity as StructuredProperty } from '../../../types.generated'; +import { Entity, IconStyleType, PreviewType } from '../Entity'; +import { getDataForEntityType } from '../shared/containers/profile/utils'; +import { urlEncodeUrn } from '../shared/utils'; + +const PreviewPropIcon = styled(TableIcon)` + font-size: 20px; +`; + +/** + * Definition of the DataHub Structured Property entity. + */ +export class StructuredPropertyEntity implements Entity { + type: EntityType = EntityType.StructuredProperty; + + icon = (fontSize?: number, styleType?: IconStyleType, color?: string) => { + if (styleType === IconStyleType.TAB_VIEW) { + return ; + } + + if (styleType === IconStyleType.HIGHLIGHT) { + return ; + } + + return ( + + ); + }; + + isSearchEnabled = () => false; + + isBrowseEnabled = () => false; + + isLineageEnabled = () => false; + + getAutoCompleteFieldName = () => 'name'; + + getGraphName = () => 'structuredProperty'; + + getPathName: () => string = () => this.getGraphName(); + + getCollectionName: () => string = () => 'Structured Properties'; + + getEntityName: () => string = () => 'Structured Property'; + + renderProfile: (urn: string) => JSX.Element = (_urn) =>
; // not used right now + + renderPreview = (previewType: PreviewType, data: StructuredProperty) => ( + } + typeIcon={this.icon(14, IconStyleType.ACCENT)} + previewType={previewType} + /> + ); + + renderSearch = (result: SearchResult) => { + return this.renderPreview(PreviewType.SEARCH, result.entity as StructuredProperty); + }; + + displayName = (data: StructuredProperty) => { + return data.definition?.displayName || data.definition?.qualifiedName || data.urn; + }; + + getGenericEntityProperties = (entity: StructuredProperty) => { + return getDataForEntityType({ data: entity, entityType: this.type, getOverrideProperties: (data) => data }); + }; + + supportedCapabilities = () => { + return new Set([]); + }; +} diff --git a/datahub-web-react/src/app/entity/tag/Tag.tsx b/datahub-web-react/src/app/entity/tag/Tag.tsx index 6f0839e5f812be..d3c5b079660993 100644 --- a/datahub-web-react/src/app/entity/tag/Tag.tsx +++ b/datahub-web-react/src/app/entity/tag/Tag.tsx @@ -46,6 +46,8 @@ export class TagEntity implements Entity { getAutoCompleteFieldName = () => 'name'; + getGraphName = () => 'tag'; + getPathName: () => string = () => 'tag'; getCollectionName: () => string = () => 'Tags'; diff --git a/datahub-web-react/src/app/entity/user/User.tsx b/datahub-web-react/src/app/entity/user/User.tsx index ec1c5fbdc86980..058349f83eaecb 100644 --- a/datahub-web-react/src/app/entity/user/User.tsx +++ b/datahub-web-react/src/app/entity/user/User.tsx @@ -39,6 +39,8 @@ export class UserEntity implements Entity { getAutoCompleteFieldName = () => 'username'; + getGraphName: () => string = () => 'corpuser'; + getPathName: () => string = () => 'user'; getEntityName = () => 'Person'; diff --git a/datahub-web-react/src/app/govern/structuredProperties/AdvancedOptions.tsx b/datahub-web-react/src/app/govern/structuredProperties/AdvancedOptions.tsx new file mode 100644 index 00000000000000..620143258ef5f5 --- /dev/null +++ b/datahub-web-react/src/app/govern/structuredProperties/AdvancedOptions.tsx @@ -0,0 +1,64 @@ +import { Icon, Input, Text, Tooltip } from '@components'; +import { Collapse, Form } from 'antd'; +import React from 'react'; +import { CollapseHeader, FlexContainer, InputLabel, StyledCollapse } from './styledComponents'; + +interface Props { + isEditMode: boolean; +} + +const AdvancedOptions = ({ isEditMode }: Props) => { + return ( + ( + + )} + expandIconPosition="end" + defaultActiveKey={[]} + > + + + Advanced Options + + + } + forceRender + > + + + Qualified Name + + + + + + + + + + + + + ); +}; + +export default AdvancedOptions; diff --git a/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesDrawer.tsx b/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesDrawer.tsx new file mode 100644 index 00000000000000..f1dccb6db0c22c --- /dev/null +++ b/datahub-web-react/src/app/govern/structuredProperties/AllowedValuesDrawer.tsx @@ -0,0 +1,142 @@ +import { Button, Icon, Input, Text, TextArea } from '@src/alchemy-components'; +import { AllowedValue } from '@src/types.generated'; +import { Form, FormInstance } from 'antd'; +import { Tooltip } from '@components'; +import React, { useEffect, useRef } from 'react'; +import { + AddButtonContainer, + DeleteIconContainer, + FieldGroupContainer, + FormContainer, + InputLabel, + StyledDivider, + ValuesContainer, +} from './styledComponents'; +import { PropValueField } from './utils'; + +interface Props { + showAllowedValuesDrawer: boolean; + propType: PropValueField; + allowedValues: AllowedValue[] | undefined; + isEditMode: boolean; + noOfExistingValues: number; + form: FormInstance; +} + +const AllowedValuesDrawer = ({ + showAllowedValuesDrawer, + propType, + allowedValues, + isEditMode, + noOfExistingValues, + form, +}: Props) => { + useEffect(() => { + form.setFieldsValue({ allowedValues: allowedValues || [{}] }); + }, [form, showAllowedValuesDrawer, allowedValues]); + + const containerRef = useRef(null); + + // Scroll to the bottom to show the newly added fields + const scrollToBottom = () => { + if (containerRef.current) { + containerRef.current.scrollTop = containerRef.current.scrollHeight; + } + }; + + return ( +
+ + {(fields, { add, remove }) => ( + + {fields.length > 0 && ( + + {fields.map((field, index) => { + const isExisting = isEditMode && index < noOfExistingValues; + + return ( + + + Value + + * + + + + + + + + +